def compute_images(self):
"""
Compute images through decoder.
"""
assert self.model.training is False
num_batches = int(math.ceil(self.perturbations.shape[0]/self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0])
batch_code = common.torch.as_variable(self.perturbation_codes[b_start: b_end], self.args.use_gpu)
batch_theta = common.torch.as_variable(self.perturbation_theta[b_start: b_end], self.args.use_gpu)
if isinstance(self.model, models.SelectiveDecoder):
self.model.set_code(batch_code)
theta_images = self.model(batch_theta)
batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu)
perturbation_images = self.model(batch_perturbation)
if b % 100:
log('[Testing] %d' % b)
theta_images = numpy.squeeze(numpy.transpose(theta_images.cpu().detach().numpy(), (0, 2, 3, 1)))
self.theta_images = common.numpy.concatenate(self.theta_images, theta_images)
perturbation_images = numpy.squeeze(numpy.transpose(perturbation_images.cpu().detach().numpy(), (0, 2, 3, 1)))
self.perturbation_images = common.numpy.concatenate(self.perturbation_images, perturbation_images)
self.theta_images = self.theta_images.reshape((self.theta_images.shape[0], -1))
self.perturbation_images = self.perturbation_images.reshape((self.perturbation_images.shape[0], -1))
def compute_images(self):
"""
Compute images through decoder.
"""
assert self.model.training is False
num_batches = int(
math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.perturbations.shape[0])
batch_fonts = self.perturbation_codes[b_start:b_end, 1]
batch_classes = self.perturbation_codes[b_start:b_end, 2]
batch_code = numpy.concatenate(
(common.numpy.one_hot(batch_fonts, self.N_font),
common.numpy.one_hot(batch_classes, self.N_class)),
axis=1).astype(numpy.float32)
batch_code = common.torch.as_variable(batch_code,
self.args.use_gpu)
batch_theta = common.torch.as_variable(
self.perturbation_theta[b_start:b_end], self.args.use_gpu)
theta_images = self.model(batch_code, batch_theta)
batch_perturbation = common.torch.as_variable(
self.perturbations[b_start:b_end], self.args.use_gpu)
perturbation_images = self.model(batch_code, batch_perturbation)
if b % 100:
log('[Testing] %d' % b)
theta_images = numpy.squeeze(
numpy.transpose(theta_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
self.theta_images = common.numpy.concatenate(
self.theta_images, theta_images)
perturbation_images = numpy.squeeze(
numpy.transpose(perturbation_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
self.perturbation_images = common.numpy.concatenate(
self.perturbation_images, perturbation_images)
self.theta_images = self.theta_images.reshape(
(self.theta_images.shape[0], -1))
self.perturbation_images = self.perturbation_images.reshape(
(self.perturbation_images.shape[0], -1))
def compute_images(self):
"""
Compute images.
"""
assert self.test_codes is not None
num_batches = int(
math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.perturbations.shape[0])
batch_codes = common.torch.as_variable(
self.test_codes[b_start:b_end], self.args.use_gpu)
if isinstance(self.model, models.SelectiveDecoder):
self.model.set_code(batch_codes)
batch_perturbation = common.torch.as_variable(
self.perturbations[b_start:b_end].astype(numpy.float32),
self.args.use_gpu)
perturbation_images = self.model(batch_perturbation)
if b % 100:
log('[Detection] %d' % b)
perturbation_images = numpy.squeeze(
perturbation_images.cpu().detach().numpy())
self.perturbation_images = common.numpy.concatenate(
self.perturbation_images, perturbation_images)
# Trick to perform analysis on actual images of adversarial examples.
self.perturbations = self.perturbation_images
def test_random(self):
"""
Test random.
"""
pred_images = None
codes = numpy.random.normal(
0, 1, (1000, self.args.latent_space_size)).astype(numpy.float32)
num_batches = int(math.ceil(codes.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
batch_codes = common.torch.as_variable(codes[b_start:b_end],
self.args.use_gpu)
# To get the correct images!
output_images = self.decoder(batch_codes)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
if b % 100 == 50:
log('[Testing] %d' % b)
utils.write_hdf5(self.args.random_file, pred_images)
log('[Testing] wrote %s' % self.args.random_file)
def compute_images(self):
"""
Compute images through decoder.
"""
num_batches = int(math.ceil(self.perturbations.shape[0]/self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0])
batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu)
self.model.set_image(batch_images)
batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu)
perturbation_images = self.model(batch_perturbation)
if b % 100:
log('[Testing] %d' % b)
perturbation_images = numpy.squeeze(numpy.transpose(perturbation_images.cpu().detach().numpy(), (0, 2, 3, 1)))
self.perturbation_images = common.numpy.concatenate(self.perturbation_images, perturbation_images)
self.test_images = self.test_images.reshape((self.test_images.shape[0], -1))
self.perturbation_images = self.perturbation_images.reshape((self.perturbation_images.shape[0], -1))
def load_data(self):
"""
Load data and model.
"""
self.test_images = utils.read_hdf5(self.args.test_images_file).astype(numpy.float32)
log('[Testing] read %s' % self.args.test_images_file)
# For handling both color and gray images.
if len(self.test_images.shape) < 4:
self.test_images = numpy.expand_dims(self.test_images, axis=3)
log('[Testing] no color images, adjusted size')
self.resolution = self.test_images.shape[2]
log('[Testing] resolution %d' % self.resolution)
self.train_images = utils.read_hdf5(self.args.train_images_file).astype(numpy.float32)
# !
self.train_images = self.train_images.reshape((self.train_images.shape[0], -1))
log('[Testing] read %s' % self.args.train_images_file)
self.test_codes = utils.read_hdf5(self.args.test_codes_file).astype(numpy.int)
self.test_codes = self.test_codes[:, self.args.label_index]
self.N_class = numpy.max(self.test_codes) + 1
log('[Testing] read %s' % self.args.test_codes_file)
self.accuracy = utils.read_hdf5(self.args.accuracy_file)
log('[Testing] read %s' % self.args.accuracy_file)
self.perturbations = utils.read_hdf5(self.args.perturbations_file).astype(numpy.float32)
self.N_attempts = self.perturbations.shape[0]
# First, repeat relevant data.
self.test_images = numpy.repeat(self.test_images[:self.perturbations.shape[1]], self.N_attempts, axis=0)
self.perturbation_codes = numpy.repeat(self.test_codes[:self.perturbations.shape[1]], self.N_attempts, axis=0)
self.perturbation_codes = numpy.squeeze(self.perturbation_codes)
self.accuracy = numpy.repeat(self.accuracy[:self.perturbations.shape[1]], self.N_attempts, axis=0)
# Then, reshape the perturbations!
self.perturbations = numpy.swapaxes(self.perturbations, 0, 1)
self.perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1))
assert self.perturbations.shape[1] == self.args.N_theta
log('[Testing] read %s' % self.args.perturbations_file)
assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations'
self.success = utils.read_hdf5(self.args.success_file)
self.success = numpy.swapaxes(self.success, 0, 1)
self.success = self.success.reshape((self.success.shape[0] * self.success.shape[1]))
log('[Testing] read %s' % self.args.success_file)
log('[Testing] using %d input channels' % self.test_images.shape[3])
assert self.args.N_theta > 0 and self.args.N_theta <= 9
decoder = models.STNDecoder(self.args.N_theta)
# decoder.eval()
log('[Testing] set up STN decoder')
self.model = decoder
def test(self):
"""
Test classifier to identify valid samples to attack.
"""
num_batches = int(
math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.perturbations.shape[0])
batch_fonts = self.test_fonts[b_start:b_end]
batch_classes = self.test_classes[b_start:b_end]
batch_code = numpy.concatenate(
(common.numpy.one_hot(batch_fonts, self.N_font),
common.numpy.one_hot(batch_classes, self.N_class)),
axis=1).astype(numpy.float32)
batch_inputs = common.torch.as_variable(
self.perturbations[b_start:b_end], self.args.use_gpu)
batch_code = common.torch.as_variable(batch_code,
self.args.use_gpu)
# This basically allows to only optimize over theta, keeping the font/class code fixed.
self.model.set_code(batch_code)
output_images = self.model(batch_inputs)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
self.perturbation_images = common.numpy.concatenate(
self.perturbation_images, output_images)
if b % 100 == 0:
log('[Testing] computing perturbation images %d' % b)
utils.makedir(os.path.dirname(self.args.perturbation_images_file))
if len(self.perturbation_images.shape) > 3:
self.perturbation_images = self.perturbation_images.reshape(
self.N_samples, self.N_attempts,
self.perturbation_images.shape[1],
self.perturbation_images.shape[2],
self.perturbation_images.shape[3])
else:
self.perturbation_images = self.perturbation_images.reshape(
self.N_samples, self.N_attempts,
self.perturbation_images.shape[1],
self.perturbation_images.shape[2])
self.perturbation_images = numpy.swapaxes(self.perturbation_images, 0,
1)
utils.write_hdf5(self.args.perturbation_images_file,
self.perturbation_images)
log('[Testing] wrote %s' % self.args.perturbation_images_file)
def test(self):
"""
Test classifier to identify valid samples to attack.
"""
num_batches = int(
math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.perturbations.shape[0])
batch_images = common.torch.as_variable(
self.test_images[b_start:b_end], self.args.use_gpu)
batch_inputs = common.torch.as_variable(
self.perturbations[b_start:b_end], self.args.use_gpu)
self.model.set_image(batch_images)
output_images = self.model(batch_inputs)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
self.perturbation_images = common.numpy.concatenate(
self.perturbation_images, output_images)
if b % 100 == 0:
log('[Testing] computing perturbation images %d' % b)
utils.makedir(os.path.dirname(self.args.perturbation_images_file))
if len(self.perturbation_images.shape) > 3:
self.perturbation_images = self.perturbation_images.reshape(
self.N_samples, self.N_attempts,
self.perturbation_images.shape[1],
self.perturbation_images.shape[2],
self.perturbation_images.shape[3])
else:
self.perturbation_images = self.perturbation_images.reshape(
self.N_samples, self.N_attempts,
self.perturbation_images.shape[1],
self.perturbation_images.shape[2])
self.perturbation_images = numpy.swapaxes(self.perturbation_images, 0,
1)
utils.write_hdf5(self.args.perturbation_images_file,
self.perturbation_images)
log('[Testing] wrote %s' % self.args.perturbation_images_file)
def sample(self):
"""
Test the model.
"""
assert self.decoder is not None
self.decoder.eval()
log('[Sampling] set decoder to eval')
images = None
theta = common.numpy.truncated_normal(
(self.args.N_samples, self.args.latent_space_size),
lower=-self.args.bound,
upper=self.args.bound).astype(numpy.float32)
theta = theta.astype(numpy.float32)
num_batches = int(math.ceil(theta.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, theta.shape[0])
batch_theta = common.torch.as_variable(theta[b_start:b_end],
self.args.use_gpu)
# Important to get the correct codes!
assert self.decoder.training is False
output_images = self.decoder(batch_theta)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
images = common.numpy.concatenate(images, output_images)
if b % 100 == 50:
log('[Sampling] %d' % b)
if self.args.images_file:
utils.write_hdf5(self.args.images_file, images)
log('[Sampling] wrote %s' % self.args.images_file)
if self.args.theta_file:
utils.write_hdf5(self.args.theta_file, theta)
log('[Sampling] wrote %s' % self.args.theta_file)
def test_interpolation(self):
"""
Test interpolation.
"""
interpolations = None
perm = numpy.random.permutation(
numpy.array(range(self.pred_codes.shape[0])))
for i in range(50):
first = self.pred_codes[i]
second = self.pred_codes[perm[i]]
linfit = scipy.interpolate.interp1d([0, 1],
numpy.vstack([first, second]),
axis=0)
interpolations = common.numpy.concatenate(
interpolations, linfit(numpy.linspace(0, 1, 10)))
pred_images = None
num_batches = int(
math.ceil(interpolations.shape[0] / self.args.batch_size))
interpolations = interpolations.astype(numpy.float32)
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
batch_codes = common.torch.as_variable(
interpolations[b_start:b_end], self.args.use_gpu)
# To get the correct images!
output_images = self.decoder(batch_codes)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
if b % 100 == 50:
log('[Testing] %d' % b)
utils.write_hdf5(self.args.interpolation_file, pred_images)
log('[Testing] wrote %s' % self.args.interpolation_file)
def compute_images(self):
"""
Compute images.
"""
assert self.test_codes is not None
num_batches = int(
math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.perturbations.shape[0])
batch_fonts = self.test_codes[b_start:b_end, 1]
batch_classes = self.test_codes[b_start:b_end, 2]
batch_code = numpy.concatenate(
(common.numpy.one_hot(batch_fonts, self.N_font),
common.numpy.one_hot(batch_classes, self.N_class)),
axis=1).astype(numpy.float32)
batch_code = common.torch.as_variable(batch_code,
self.args.use_gpu)
batch_perturbation = common.torch.as_variable(
self.perturbations[b_start:b_end].astype(numpy.float32),
self.args.use_gpu)
perturbation_images = self.model(batch_code, batch_perturbation)
if b % 100:
log('[Testing] %d' % b)
perturbation_images = numpy.squeeze(
perturbation_images.cpu().detach().numpy())
self.perturbation_images = common.numpy.concatenate(
self.perturbation_images, perturbation_images)
# Trick to perform analysis on actual images of adversarial examples.
self.perturbations = self.perturbation_images
def load_data(self):
"""
Load data and model.
"""
self.test_images = utils.read_hdf5(self.args.test_images_file).astype(numpy.float32)
log('[Testing] read %s' % self.args.test_images_file)
# For handling both color and gray images.
if len(self.test_images.shape) < 4:
self.test_images = numpy.expand_dims(self.test_images, axis=3)
log('[Testing] no color images, adjusted size')
self.resolution = self.test_images.shape[2]
log('[Testing] resolution %d' % self.resolution)
self.train_images = utils.read_hdf5(self.args.train_images_file).astype(numpy.float32)
# !
self.train_images = self.train_images.reshape((self.train_images.shape[0], -1))
log('[Testing] read %s' % self.args.train_images_file)
self.test_theta = utils.read_hdf5(self.args.test_theta_file).astype(numpy.float32)
log('[Testing] read %s' % self.args.test_theta_file)
self.train_theta = utils.read_hdf5(self.args.train_theta_file).astype(numpy.float32)
log('[Testing] read %s' % self.args.train_theta_file)
self.test_codes = utils.read_hdf5(self.args.test_codes_file).astype(numpy.int)
self.test_codes = self.test_codes[:, self.args.label_index]
self.N_class = numpy.max(self.test_codes) + 1
log('[Testing] read %s' % self.args.test_codes_file)
self.accuracy = utils.read_hdf5(self.args.accuracy_file)
log('[Testing] read %s' % self.args.accuracy_file)
self.perturbations = utils.read_hdf5(self.args.perturbations_file).astype(numpy.float32)
self.N_attempts = self.perturbations.shape[0]
assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations'
# First, repeat relevant data.
self.perturbation_theta = numpy.repeat(self.test_theta[:self.perturbations.shape[1]], self.N_attempts, axis=0)
self.perturbation_codes = numpy.repeat(self.test_codes[:self.perturbations.shape[1]], self.N_attempts, axis=0)
self.perturbation_codes = numpy.squeeze(self.perturbation_codes)
self.accuracy = numpy.repeat(self.accuracy[:self.perturbations.shape[1]], self.N_attempts, axis=0)
# Then, reshape the perturbations!
self.perturbations = numpy.swapaxes(self.perturbations, 0, 1)
self.perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1))
log('[Testing] read %s' % self.args.perturbations_file)
self.success = utils.read_hdf5(self.args.success_file)
self.success = numpy.swapaxes(self.success, 0, 1)
self.success = self.success.reshape((self.success.shape[0] * self.success.shape[1]))
log('[Testing] read %s' % self.args.success_file)
assert self.args.decoder_files
decoder_files = self.args.decoder_files.split(',')
for decoder_file in decoder_files:
assert os.path.exists(decoder_file), 'could not find %s' % decoder_file
log('[Testing] using %d input channels' % self.test_images.shape[3])
decoder_units = list(map(int, self.args.decoder_units.split(',')))
if len(decoder_files) > 1:
log('[Testing] loading multiple decoders')
decoders = []
for i in range(len(decoder_files)):
decoder = models.LearnedDecoder(self.args.latent_space_size, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2]),
architecture=self.args.decoder_architecture,
start_channels=self.args.decoder_channels,
activation=self.args.decoder_activation,
batch_normalization=not self.args.decoder_no_batch_normalization,
units=decoder_units)
state = State.load(decoder_files[i])
decoder.load_state_dict(state.model)
if self.args.use_gpu and not cuda.is_cuda(decoder):
decoder = decoder.cuda()
decoders.append(decoder)
decoder.eval()
log('[Testing] loaded %s' % decoder_files[i])
self.model = models.SelectiveDecoder(decoders, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2]))
else:
log('[Testing] loading one decoder')
decoder = models.LearnedDecoder(self.args.latent_space_size, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2]),
architecture=self.args.decoder_architecture,
start_channels=self.args.decoder_channels,
activation=self.args.decoder_activation,
batch_normalization=not self.args.decoder_no_batch_normalization,
units=decoder_units)
state = State.load(decoder_files[0])
decoder.load_state_dict(state.model)
if self.args.use_gpu and not cuda.is_cuda(decoder):
decoder = decoder.cuda()
decoder.eval()
log('[Testing] read decoder')
self.model = decoder
def compute_true(self):
"""
Compute true.
"""
assert self.test_codes is not None
num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size))
params = {
'lr': 0.09,
'lr_decay': 0.95,
'lr_min': 0.0000001,
'weight_decay': 0,
}
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0])
batch_fonts = self.test_codes[b_start: b_end, 1]
batch_classes = self.test_codes[b_start: b_end, 2]
batch_code = numpy.concatenate((common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype( numpy.float32)
batch_code = common.torch.as_variable(batch_code, self.args.use_gpu)
batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True)
batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end].astype(numpy.float32), self.args.use_gpu)
self.model.set_code(batch_code)
#output_images = self.model.forward(batch_theta)
#test_error = torch.mean(torch.mul(output_images - batch_images, output_images - batch_images))
#print(test_error.item())
#vis.mosaic('true.png', batch_images.cpu().detach().numpy()[:, 0, :, :])
#vis.mosaic('output.png', output_images.cpu().detach().numpy()[:, 0, :, :])
# print(batch_images.cpu().detach().numpy()[0])
# print(output_images.cpu().detach().numpy()[0, 0])
#_batch_images = batch_images.cpu().detach().numpy()
#_output_images = output_images.cpu().detach().numpy()[:, 0, :, :]
#test_error = numpy.max(numpy.abs(_batch_images.reshape(_batch_images.shape[0], -1) - _output_images.reshape(_output_images.shape[0], -1)), axis=1)
#print(test_error)
#test_error = numpy.mean(numpy.multiply(_batch_images - _output_images, _batch_images - _output_images), axis=1)
#print(test_error)
batch_theta = torch.nn.Parameter(batch_theta)
scheduler = ADAMScheduler([batch_theta], **params)
log('[Detection] %d: start' % b)
for t in range(100):
scheduler.update(t//10, float(t)/10)
scheduler.optimizer.zero_grad()
output_perturbation = self.model.forward(batch_theta)
error = torch.mean(torch.mul(output_perturbation - batch_perturbation, output_perturbation - batch_perturbation))
test_error = torch.mean(torch.mul(output_perturbation - batch_images, output_perturbation - batch_images))
#error.backward()
#scheduler.optimizer.step()
log('[Detection] %d: %d = %g, %g' % (b, t, error.item(), test_error.item()))
output_perturbation = numpy.squeeze(numpy.transpose(output_perturbation.cpu().detach().numpy(), (0, 2, 3, 1)))
self.projected_perturbations = common.numpy.concatenate(self.projected_perturbations, output_perturbation)
projected_perturbations = self.projected_perturbations.reshape((self.projected_perturbations.shape[0], -1))
perturbations = self.perturbations.reshape((self.perturbations.shape[0], -1))
success = numpy.logical_and(self.success >= 0, self.accuracy)
log('[Detection] %d valid attacked samples' % numpy.sum(success))
self.distances['true'] = numpy.linalg.norm(perturbations - projected_perturbations, ord=2, axis=1)
self.angles['true'] = numpy.rad2deg(common.numpy.angles(perturbations.T, projected_perturbations.T))
self.distances['true'] = self.distances['true'][success]
self.angles['true'] = self.angles['true'][success]
self.distances['test'] = numpy.zeros((numpy.sum(success)))
self.angles['test'] = numpy.zeros((numpy.sum(success)))
def test_test(self):
"""
Test on testing set.
"""
num_batches = int(
math.ceil(self.test_images.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
batch_images = common.torch.as_variable(
self.test_images[b_start:b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
# Important to get the correct codes!
output_codes, output_logvar = self.encoder(batch_images)
output_images = self.decoder(output_codes)
e = self.reconstruction_loss(batch_images, output_images)
self.reconstruction_error += e.data
self.code_mean += torch.mean(output_codes).item()
self.code_var += torch.var(output_codes).item()
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
self.pred_images = common.numpy.concatenate(
self.pred_images, output_images)
output_codes = output_codes.cpu().detach().numpy()
self.pred_codes = common.numpy.concatenate(self.pred_codes,
output_codes)
if b % 100 == 50:
log('[Testing] %d' % b)
assert self.pred_images.shape[0] == self.test_images.shape[
0], 'computed invalid number of test images'
if self.args.reconstruction_file:
utils.write_hdf5(self.args.reconstruction_file, self.pred_images)
log('[Testing] wrote %s' % self.args.reconstruction_file)
if self.args.test_theta_file:
assert self.pred_codes.shape[0] == self.test_images.shape[
0], 'computed invalid number of test codes'
utils.write_hdf5(self.args.test_theta_file, self.pred_codes)
log('[Testing] wrote %s' % self.args.test_theta_file)
threshold = 0.9
percentage = 0
# values = numpy.linalg.norm(pred_codes, ord=2, axis=1)
values = numpy.max(numpy.abs(self.pred_codes), axis=1)
while percentage < 0.9:
threshold += 0.1
percentage = numpy.sum(values <= threshold) / float(
values.shape[0])
log('[Testing] threshold %g percentage %g' %
(threshold, percentage))
log('[Testing] taking threshold %g with percentage %g' %
(threshold, percentage))
if self.args.output_directory and utils.display():
# fit = 10
# plot_file = os.path.join(self.args.output_directory, 'test_codes')
# plot.manifold(plot_file, pred_codes[::fit], None, None, 'tsne', None, title='t-SNE of Test Codes')
# log('[Testing] wrote %s' % plot_file)
for d in range(1, self.pred_codes.shape[1]):
plot_file = os.path.join(self.args.output_directory,
'test_codes_%s' % d)
plot.scatter(
plot_file,
self.pred_codes[:, 0],
self.pred_codes[:, d], (values <= threshold).astype(int),
['greater %g' % threshold,
'smaller %g' % threshold],
title='Dimensions 0 and %d of Test Codes' % d)
log('[Testing] wrote %s' % plot_file)
self.reconstruction_error /= num_batches
log('[Testing] reconstruction error %g' % self.reconstruction_error)
def test(self, epoch):
"""
Test the model.
:param epoch: current epoch
:type epoch: int
"""
self.encoder.eval()
log('[Training] %d set encoder to eval' % epoch)
self.decoder.eval()
log('[Training] %d set decoder to eval' % epoch)
self.classifier.eval()
log('[Training] %d set classifier to eval' % epoch)
latent_loss = 0
reconstruction_loss = 0
reconstruction_error = 0
decoder_loss = 0
discriminator_loss = 0
mean = 0
var = 0
logvar = 0
pred_images = None
pred_codes = None
num_batches = int(
math.ceil(self.test_images.shape[0] / self.args.batch_size))
assert self.encoder.training is False
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
batch_images = common.torch.as_variable(
self.test_images[b_start:b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_mu, output_logvar = self.encoder(batch_images)
output_images = self.decoder(output_mu)
output_real_classes = self.classifier(batch_images)
output_reconstructed_classes = self.classifier(output_images)
# Latent loss.
e = self.latent_loss(output_mu, output_logvar)
latent_loss += e.item()
# Reconstruction loss.
e = self.reconstruction_loss(batch_images, output_images)
reconstruction_loss += e.item()
# Reconstruction error.
e = self.reconstruction_error(batch_images, output_images)
reconstruction_error += e.item()
e = self.decoder_loss(output_reconstructed_classes)
decoder_loss += e.item()
# Adversarial loss.
e = self.discriminator_loss(output_real_classes,
output_reconstructed_classes)
discriminator_loss += e.item()
mean += torch.mean(output_mu).item()
var += torch.var(output_mu).item()
logvar += torch.mean(output_logvar).item()
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
output_codes = output_mu.cpu().detach().numpy()
pred_codes = common.numpy.concatenate(pred_codes, output_codes)
utils.write_hdf5(self.args.reconstruction_file, pred_images)
log('[Training] %d: wrote %s' % (epoch, self.args.reconstruction_file))
if utils.display():
png_file = self.args.reconstruction_file + '.%d.png' % epoch
if epoch == 0:
vis.mosaic(png_file, self.test_images[:225], 15, 5, 'gray', 0,
1)
else:
vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
latent_loss /= num_batches
reconstruction_loss /= num_batches
reconstruction_error /= num_batches
decoder_loss /= num_batches
discriminator_loss /= num_batches
mean /= num_batches
var /= num_batches
logvar /= num_batches
log('[Training] %d: test %g (%g) %g (%g, %g, %g)' %
(epoch, reconstruction_loss, reconstruction_error, latent_loss,
mean, var, logvar))
log('[Training] %d: test %g %g' %
(epoch, decoder_loss, discriminator_loss))
num_batches = int(
math.ceil(self.train_images.shape[0] / self.args.batch_size))
iteration = epoch * num_batches
self.test_statistics = numpy.vstack(
(self.test_statistics,
numpy.array([
iteration, iteration * self.args.batch_size,
min(num_batches, iteration),
min(num_batches, iteration) * self.args.batch_size,
reconstruction_loss, reconstruction_error, latent_loss, mean,
var, logvar, decoder_loss, discriminator_loss
])))
pred_images = None
if self.random_codes is None:
self.random_codes = common.numpy.truncated_normal(
(1000, self.args.latent_space_size)).astype(numpy.float32)
num_batches = int(
math.ceil(self.random_codes.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
if b_start >= b_end: break
batch_codes = common.torch.as_variable(
self.random_codes[b_start:b_end], self.args.use_gpu)
output_images = self.decoder(batch_codes)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
utils.write_hdf5(self.args.random_file, pred_images)
log('[Training] %d: wrote %s' % (epoch, self.args.random_file))
if utils.display() and epoch > 0:
png_file = self.args.random_file + '.%d.png' % epoch
vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
interpolations = None
perm = numpy.random.permutation(numpy.array(range(
pred_codes.shape[0])))
for i in range(50):
first = pred_codes[i]
second = pred_codes[perm[i]]
linfit = scipy.interpolate.interp1d([0, 1],
numpy.vstack([first, second]),
axis=0)
interpolations = common.numpy.concatenate(
interpolations, linfit(numpy.linspace(0, 1, 10)))
pred_images = None
num_batches = int(
math.ceil(interpolations.shape[0] / self.args.batch_size))
interpolations = interpolations.astype(numpy.float32)
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
if b_start >= b_end: break
batch_codes = common.torch.as_variable(
interpolations[b_start:b_end], self.args.use_gpu)
output_images = self.decoder(batch_codes)
output_images = numpy.squeeze(
numpy.transpose(output_images.cpu().detach().numpy(),
(0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
if b % 100 == 50:
log('[Testing] %d' % b)
utils.write_hdf5(self.args.interpolation_file, pred_images)
log('[Testing] wrote %s' % self.args.interpolation_file)
if utils.display() and epoch > 0:
png_file = self.args.interpolation_file + '.%d.png' % epoch
vis.mosaic(png_file, pred_images[:100], 10, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
def attack(self):
"""
Test the model.
"""
assert self.model is not None
assert self.model.training is False
assert self.test_images.shape[0] == self.test_codes.shape[0], 'number of samples has to match'
concatenate_axis = -1
if os.path.exists(self.args.perturbations_file) and os.path.exists(self.args.success_file):
self.original_perturbations = utils.read_hdf5(self.args.perturbations_file)
if self.test_images.shape[3] > 1:
assert len(self.original_perturbations.shape) == 5
else:
assert len(self.original_perturbations.shape) == 4
log('[Attack] read %s' % self.args.perturbations_file)
self.original_success = utils.read_hdf5(self.args.success_file)
log('[Attack] read %s' % self.args.success_file)
assert self.original_perturbations.shape[0] == self.original_success.shape[0]
assert self.original_perturbations.shape[1] == self.original_success.shape[1]
assert self.original_perturbations.shape[2] == self.test_images.shape[1]
assert self.original_perturbations.shape[3] == self.test_images.shape[2]#
if self.original_perturbations.shape[1] >= self.args.max_samples and self.original_perturbations.shape[0] >= self.args.max_attempts:
log('[Attack] found %d attempts, %d samples, requested no more' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1]))
return
elif self.original_perturbations.shape[0] == self.args.max_attempts or self.original_perturbations.shape[1] == self.args.max_samples:
if self.original_perturbations.shape[0] == self.args.max_attempts:
self.test_images = self.test_images[self.original_perturbations.shape[1]:]
self.test_codes = self.test_codes[self.original_perturbations.shape[1]:]
self.args.max_samples = self.args.max_samples - self.original_perturbations.shape[1]
concatenate_axis = 1
log('[Attack] found %d attempts with %d perturbations, computing %d more perturbations' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1], self.args.max_samples))
elif self.original_perturbations.shape[1] == self.args.max_samples:
self.args.max_attempts = self.args.max_attempts - self.original_perturbations.shape[0]
concatenate_axis = 0
log('[Attack] found %d attempts with %d perturbations, computing %d more attempts' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1], self.args.max_attempts))
# can't squeeze here!
if self.test_images.shape[3] > 1:
self.perturbations = numpy.zeros((self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2], self.test_images.shape[3]))
else:
self.perturbations = numpy.zeros((self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2]))
self.success = numpy.ones((self.args.max_attempts, self.args.max_samples), dtype=int) * -1
if self.args.attack.find('Batch') >= 0:
batch_size = min(self.args.batch_size, self.args.max_samples)
else:
batch_size = 1
objective = self.objective_class()
num_batches = int(math.ceil(self.args.max_samples/batch_size))
for i in range(num_batches): # self.test_images.shape[0]
if i*batch_size == self.args.max_samples:
break
i_start = i*batch_size
i_end = min((i+1)*batch_size, self.args.max_samples)
batch_images = common.torch.as_variable(self.test_images[i_start: i_end], self.args.use_gpu)
batch_classes = common.torch.as_variable(numpy.array(self.test_codes[i_start: i_end]), self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
t = 0
while t < self.args.max_attempts:
attack = self.setup_attack(batch_images, batch_classes)
success, perturbations, probabilities, norm, _ = attack.run(objective)
assert not numpy.any(perturbations != perturbations), perturbations
# Note that we save the perturbed image, not only the perturbation!
self.perturbations[t][i_start: i_end] = numpy.squeeze(numpy.transpose(perturbations + batch_images.cpu().numpy(), (0, 2, 3, 1)))
self.success[t][i_start: i_end] = success
# IMPORTANT: The adversarial examples are not considering whether the classifier is
# actually correct to start with.
t += 1
log('[Attack] %d: completed' % i)
if concatenate_axis >= 0:
if self.perturbations.shape[0] == self.args.max_attempts:
self.perturbations = numpy.concatenate((self.original_perturbations, self.perturbations), axis=concatenate_axis)
self.success = numpy.concatenate((self.original_success, self.success), axis=concatenate_axis)
log('[Attack] concatenated')
utils.write_hdf5(self.args.perturbations_file, self.perturbations)
log('[Attack] wrote %s' % self.args.perturbations_file)
utils.write_hdf5(self.args.success_file, self.success)
log('[Attack] wrote %s' % self.args.success_file)
def compute_appr(self):
"""
Compute approximate.
"""
assert self.test_codes is not None
num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0])
batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu)
batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True)
batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end].astype(numpy.float32), self.args.use_gpu)
if isinstance(self.model, models.SelectiveDecoder):
self.model.set_code(batch_classes)
batch_theta = torch.nn.Parameter(batch_theta)
optimizer = torch.optim.Adam([batch_theta], lr=0.1)
log('[Detection] %d: start' % b)
for t in range(100):
optimizer.zero_grad()
output_perturbation = self.model.forward(batch_theta)
error = torch.mean(torch.mul(output_perturbation - batch_perturbation, output_perturbation - batch_perturbation))
error.backward()
optimizer.step()
log('[Detection] %d: %d = %g' % (b, t, error.item()))
output_perturbation = numpy.squeeze(output_perturbation.cpu().detach().numpy())
self.projected_perturbations = common.numpy.concatenate(self.projected_perturbations, output_perturbation)
batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True)
batch_images = common.torch.as_variable(self.test_images[b_start: b_end].astype(numpy.float32), self.args.use_gpu)
batch_theta = torch.nn.Parameter(batch_theta)
optimizer = torch.optim.Adam([batch_theta], lr=0.5)
log('[Detection] %d: start' % b)
for t in range(100):
optimizer.zero_grad()
output_images = self.model.forward(batch_theta)
error = torch.mean(torch.mul(output_images - batch_images, output_images - batch_images))
error.backward()
optimizer.step()
log('[Detection] %d: %d = %g' % (b, t, error.item()))
output_images = numpy.squeeze(output_images.cpu().detach().numpy())
self.projected_test_images = common.numpy.concatenate(self.projected_test_images, output_images)
projected_perturbations = self.projected_perturbations.reshape((self.projected_perturbations.shape[0], -1))
projected_test_images = self.projected_test_images.reshape((self.projected_test_images.shape[0], -1))
perturbations = self.perturbations.reshape((self.perturbations.shape[0], -1))
test_images = self.test_images.reshape((self.test_images.shape[0], -1))
success = numpy.logical_and(self.success >= 0, self.accuracy)
log('[Detection] %d valid attacked samples' % numpy.sum(success))
self.distances['true'] = numpy.linalg.norm(perturbations - projected_perturbations, ord=2, axis=1)
self.angles['true'] = numpy.rad2deg(common.numpy.angles(perturbations.T, projected_perturbations.T))
self.distances['true'] = self.distances['true'][success]
self.angles['true'] = self.angles['true'][success]
self.distances['test'] = numpy.linalg.norm(test_images - projected_test_images, ord=2, axis=1)
self.angles['test'] = numpy.rad2deg(common.numpy.angles(test_images.T, projected_test_images.T))
self.distances['test'] = self.distances['test'][success]
self.angles['test'] = self.angles['test'][success]
def attack(self):
"""
Test the model.
"""
assert self.model is not None
assert self.model.training is False
if self.args.attack.find('Batch') >= 0:
batch_size = min(self.args.batch_size, self.args.max_samples)
else:
batch_size = 1
objective = self.objective_class()
num_batches = int(math.ceil(self.args.max_samples / batch_size))
# can't squeeze here!
if self.test_images.shape[3] > 1:
self.perturbations = numpy.zeros(
(self.args.max_attempts, self.args.max_samples,
self.test_images.shape[1], self.test_images.shape[2],
self.test_images.shape[3]))
else:
self.perturbations = numpy.zeros(
(self.args.max_attempts, self.args.max_samples,
self.test_images.shape[1], self.test_images.shape[2]))
self.success = numpy.ones(
(self.args.max_attempts, self.args.max_samples), dtype=int) * -1
self.probabilities = numpy.zeros(
(self.args.max_attempts, self.args.max_samples, self.N_class))
for i in range(num_batches): # self.test_images.shape[0]
if i * batch_size == self.args.max_samples:
break
i_start = i * batch_size
i_end = min((i + 1) * batch_size, self.args.max_samples)
batch_images = numpy.random.randint(0,
255,
size=[batch_size] +
self.test_images.shape[1:])
batch_images = common.torch.as_variable(batch_images,
self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
batch_classes = common.torch.as_variable(
numpy.random.randint(0,
self.N_class - 1,
size=(batch_images.size(0))),
self.args.use_gpu)
t = 0
while t < self.args.max_attempts:
attack = self.setup_attack(batch_images, batch_classes)
success, perturbations, probabilities, norm, _ = attack.run(
objective)
assert not numpy.any(
perturbations != perturbations), perturbations
# Note that we save the perturbed image, not only the perturbation!
self.perturbations[t][i_start:i_end] = numpy.squeeze(
numpy.transpose(perturbations + batch_images.cpu().numpy(),
(0, 2, 3, 1)))
self.success[t][i_start:i_end] = success
self.probabilities[t][i_start:i_end] = probabilities
# IMPORTANT: The adversarial examples are not considering whether the classifier is
# actually correct to start with.
t += 1
log('[Attack] %d: completed' % i)
utils.write_hdf5(self.args.perturbations_file, self.perturbations)
log('[Attack] wrote %s' % self.args.perturbations_file)
utils.write_hdf5(self.args.success_file, self.success)
log('[Attack] wrote %s' % self.args.success_file)
utils.write_hdf5(self.args.probabilities_file, self.probabilities)
log('[Attack] wrote %s' % self.args.probabilities_file)
