def update_core(self):
batch = self.get_iterator('main').next()
input_var = Variable(self.converter(batch, self.device), volatile='on')
content_features = extract({'data': input_var}, self.D,
self.args.content_layers)
content_features = {
key: value[0]
for key, value in content_features.items()
}
for _, value in content_features.items():
value.volatile = 'off'
input_var.volatile = 'off'
output_var = self.G(input_var)
ouput_features = extract({'data': output_var}, self.D, self.layers)
optimizer = self.get_optimizer('main')
optimizer.update(self.loss, ouput_features, content_features,
output_var)
def __call__(self, x, img_real):
if type(img_real) != chainer.variable.Variable: # if validation set
img_real = Variable(img_real)
## Compute latent space from BOLD
z = self.predictor(x)
## Generate images from latent space
img_fake = self.pretrained_gan.generate_img_from_z(z)
img_fake = F.clip(img_fake, -1.0, 1.0) # avoid slight overflow of values (after tanh, up to 1.07)
img_fake.volatile = 'OFF' ; img_real.volatile = 'OFF' # workaround an issue during validation
## Get activations of perceptual features
_, layer_activations_fake = self.featnet( img_fake, train=False, return_activations=True )
_, layer_activations_real = self.featnet( img_real, train=False, return_activations=True )
# Note that featnet can also return the non-softmaxed final layer activations (=the classes, here in _ ).
# Got some bizarre (and no better) results for natural images when also including a class-matching loss.
# But (as mentioned in the paper): A loss on higher layers of a convnet trained with a discrete set of
# classes (such as ImageNet classes) may *restrict* your reconstructions to these classes, which is
# not desired. Computing a loss within a continuous semantic space may be a solution here.
## Compute perceptual losses
loss = 0.0
if self.featnet != None:
for layer_idx in ['pixel'] + args.featn_layers:
if layer_idx == 'pixel':
# compute pixel loss l_px
loss_px = args.lambda_pixel * (
F.mean_absolute_error( F.resize_images(img_fake,
(args.small_img_dims,args.small_img_dims)),
F.resize_images(img_real,
(args.small_img_dims,args.small_img_dims)) ) )
loss += loss_px
else:
layer_idx = int(layer_idx)
activ_fake_pos = F.hard_sigmoid( layer_activations_fake[layer_idx]*3.0 - 3.*args.featthre )
activ_real_pos = F.hard_sigmoid( layer_activations_real[layer_idx]*3.0 - 3.*args.featthre )
# using hard_sigmoid for a differentiable binarization at threshold 1.0
if int(layer_idx) == 0: # negative feature activations only make sense for conv1
activ_fake_neg = F.hard_sigmoid( -1.0*layer_activations_fake[layer_idx]*3.0 - 3.*args.featthre )
activ_real_neg = F.hard_sigmoid( -1.0*layer_activations_real[layer_idx]*3.0 - 3.*args.featthre )
mask_real = (activ_real_pos.data + activ_real_neg.data) > 0
else: # only use positive activations
mask_real = activ_real_pos.data > 0
loss_pr_neg = 0
if np.sum(mask_real[:]) > 0.0: # if there are any activations above 1.0
# compute l_l,m
loss_mag = args.lambda_magnitude * (
F.mean_squared_error(layer_activations_fake[layer_idx][mask_real],
layer_activations_real[layer_idx][mask_real]) )
else: # warn and set magnitude loss to 0.0 (does not happen)
loss_mag = 0.0
print "Warning: No magnitude loss"
loss += loss_mag
report({'loss': loss}, self)
# Use this code to check whether gradients were computed:
#self.predictor.l1.cleargrads()
#loss.backward()
#print "Gradients: ", self.predictor.l1.W.grad
# Do this for all new loss terms.
return loss
