def loss_dis(self, dis, y_fake, y_real):
batchsize = len(y_fake)
L1 = F.sum(F.softplus(-y_real)) / batchsize
L2 = F.sum(F.softplus(y_fake)) / batchsize
loss = L1 + L2
chainer.report({'loss': loss}, dis)
return loss
def loss_dec(self, dec, x_out, t_out, y_out, lam1=100, lam2=1):
batchsize, _, w, h = y_out.data.shape
loss_rec = lam1*(F.mean_absolute_error(x_out, t_out))
loss_adv = lam2*F.sum(F.softplus(-y_out)) / batchsize / w / h
loss = loss_rec + loss_adv
chainer.report({'loss': loss}, dec)
return loss
def test_report(self):
reporter = chainer.Reporter()
with reporter:
chainer.report({'x': 1})
observation = reporter.observation
self.assertIn('x', observation)
self.assertEqual(observation['x'], 1)
def forward(self, xs, ys):
xs = [x[::-1] for x in xs]
eos = self.xp.array([EOS], numpy.int32)
ys_in = [F.concat([eos, y], axis=0) for y in ys]
ys_out = [F.concat([y, eos], axis=0) for y in ys]
# Both xs and ys_in are lists of arrays.
exs = sequence_embed(self.embed_x, xs)
eys = sequence_embed(self.embed_y, ys_in)
batch = len(xs)
# None represents a zero vector in an encoder.
hx, cx, _ = self.encoder(None, None, exs)
_, _, os = self.decoder(hx, cx, eys)
# It is faster to concatenate data before calculating loss
# because only one matrix multiplication is called.
concat_os = F.concat(os, axis=0)
concat_ys_out = F.concat(ys_out, axis=0)
loss = F.sum(F.softmax_cross_entropy(
self.W(concat_os), concat_ys_out, reduce='no')) / batch
chainer.report({'loss': loss}, self)
n_words = concat_ys_out.shape[0]
perp = self.xp.exp(loss.array * batch / n_words)
chainer.report({'perp': perp}, self)
return loss
def loss_dis(self, dis, y_in, y_out):
batchsize,_,w,h = y_in.data.shape
L1 = F.sum(F.softplus(-y_in)) / batchsize / w / h
L2 = F.sum(F.softplus(y_out)) / batchsize / w / h
loss = L1 + L2
chainer.report({'loss': loss}, dis)
return loss
def __call__(self, x):
self.model.train = False
Wh = chainer.Variable(self.Wh_data)
Ww = chainer.Variable(self.Ww_data)
tp = chainer.Variable(self.truth_probabiliry)
t = chainer.Variable(self.t_dummy)
model_loss = self.model(self.img(), t)
#p = self.getprobability(model_loss)
a = self.getbeforesoftmax(model_loss)
#print(a.data, F.get_item(a, (0, self.label)).data)
#print(F.sum(p**2).data)
#p = (1 / F.sum(p**2)) .* (p**2)
#ce = self.getCrossEntropy(model_loss)
#print(t.data, ce.data, p.data, tp.data)
#class_mse = ce#F.mean_squared_error(p, tp)
#class_mse = F.sum(-F.log(p**2 / F.sum(p**2).data) * tp)
#activation = -F.sum(a * tp)
activation = -F.get_item(a, (0, self.label))
#activation = -F.get_item(a, (0, self.label)) / F.sqrt(F.sum(a**2) - F.get_item(a, (0, self.label))**2)
#activation = -F.get_item(a, (0, self.label)) + (F.sum(a**2) - F.get_item(a, (0, self.label))**2) / (a.data.shape[1] - 1)
#class_mse = F.sum(-F.log(p) * tp)
tv = self.tv_norm(self.img(), Wh, Ww) / np.prod(self.img().data.shape[1:])
lp = (F.sum(self.img()**self.p) ** (1./self.p)) / np.prod(self.img().data.shape[1:])
loss = self.lambda_a * activation + self.lambda_tv * tv + self.lambda_lp * lp
chainer.report({'inv_loss': loss, 'activation': activation, 'tv': tv, 'lp': lp}, self)
#print('inverter', x.data, class_mse.data, tv.data)
return loss
def __call__(self, x, t):
x.data = x.data.reshape((-1, 1)).astype(np.float32)
t.data = t.data.reshape((-1, 1)).astype(np.float32)
y = self.predictor(x)
loss = F.mean_squared_error(y, t)
report({'loss':loss}, self)
return loss
def test_report_with_observer(self):
reporter = chainer.Reporter()
observer = object()
reporter.add_observer('o', observer)
with reporter:
chainer.report({'x': 1}, observer)
observation = reporter.observation
self.assertIn('o/x', observation)
self.assertEqual(observation['o/x'], 1)
def __call__(self, x, t):
h = F.max_pooling_2d(F.relu(self.mlpconv1(x)), 3, stride=2)
h = F.max_pooling_2d(F.relu(self.mlpconv2(h)), 3, stride=2)
h = F.max_pooling_2d(F.relu(self.mlpconv3(h)), 3, stride=2)
h = self.mlpconv4(F.dropout(h, train=self.train))
h = F.reshape(F.average_pooling_2d(h, 6), (x.data.shape[0], 1000))
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def forward(self, x, t):
h = F.max_pooling_2d(F.relu(self.mlpconv1(x)), 3, stride=2)
h = F.max_pooling_2d(F.relu(self.mlpconv2(h)), 3, stride=2)
h = F.max_pooling_2d(F.relu(self.mlpconv3(h)), 3, stride=2)
h = self.mlpconv4(F.dropout(h))
h = F.reshape(F.average_pooling_2d(h, 6), (len(x), 1000))
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def __call__(self, input_ids, input_mask, token_type_ids, labels):
output_layer = self.bert.get_pooled_output(
input_ids,
input_mask,
token_type_ids)
output_layer = F.dropout(output_layer, 0.1)
logits = self.output(output_layer)
loss = F.softmax_cross_entropy(logits, labels)
chainer.report({'loss': loss.array}, self)
chainer.report({'accuracy': F.accuracy(logits, labels)}, self)
return loss
def test_report_scope(self):
reporter = chainer.Reporter()
observation = {}
with reporter:
with chainer.report_scope(observation):
chainer.report({'x': 1})
self.assertIn('x', observation)
self.assertEqual(observation['x'], 1)
self.assertNotIn('x', reporter.observation)
def __call__(self, x, t):
outputs, = super(Caffemodel, self)(inputs={'data': x}, outputs=self.outputlayers)
losses = [for x in zip(outputs, self.outputlayers) if x[1] in self.losses]
before_softmax = [for x in zip(outputs, self.outputlayers) if x[1] in self.before_softmax]
reports = dict(zip(self.losses, losses))
# calculate accuracy from fc layer
reports.update(dict(zip(
['accuracy/' + s for s in self.before_softmax],
[F.accuracy(h, t) for h in before_softmax])))
chainer.report(reports, self)
def __call__(self, x, t):
h = self.bn1(self.conv1(x))
h = F.max_pooling_2d(F.relu(h), 3, stride=2)
h = self.res2(h)
h = self.res3(h)
h = self.res4(h)
h = self.res5(h)
h = F.average_pooling_2d(h, 7, stride=1)
h = self.fc(h)
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def compute_loss(self, input_ids, input_mask, token_type_ids,
start_positions, end_positions):
(start_logits, end_logits) = self.__call__(
input_ids, input_mask, token_type_ids)
start_loss = F.softmax_cross_entropy(start_logits, start_positions)
end_loss = F.softmax_cross_entropy(end_logits, end_positions)
total_loss = (start_loss + end_loss) / 2.0
chainer.report({'loss': total_loss.array}, self)
accuracy = (check_answers(start_logits, start_positions) *
check_answers(end_logits, end_positions, start_positions)).mean()
chainer.report({'accuracy': accuracy}, self)
return total_loss
def __call__(self, x, t):
h = F.relu(self.conv1(x))
h = F.local_response_normalization(
F.max_pooling_2d(h, 3, stride=2), n=5)
h = F.relu(self.conv2_reduce(h))
h = F.relu(self.conv2(h))
h = F.max_pooling_2d(
F.local_response_normalization(h, n=5), 3, stride=2)
h = self.inc3a(h)
h = self.inc3b(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = self.inc4a(h)
l = F.average_pooling_2d(h, 5, stride=3)
l = F.relu(self.loss1_conv(l))
l = F.relu(self.loss1_fc1(l))
l = self.loss1_fc2(l)
loss1 = F.softmax_cross_entropy(l, t)
h = self.inc4b(h)
h = self.inc4c(h)
h = self.inc4d(h)
l = F.average_pooling_2d(h, 5, stride=3)
l = F.relu(self.loss2_conv(l))
l = F.relu(self.loss2_fc1(l))
l = self.loss2_fc2(l)
loss2 = F.softmax_cross_entropy(l, t)
h = self.inc4e(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = self.inc5a(h)
h = self.inc5b(h)
h = F.average_pooling_2d(h, 7, stride=1)
h = self.loss3_fc(F.dropout(h, 0.4))
loss3 = F.softmax_cross_entropy(h, t)
loss = 0.3 * (loss1 + loss2) + loss3
accuracy = F.accuracy(h, t)
chainer.report({
'loss': loss,
'loss1': loss1,
'loss2': loss2,
'loss3': loss3,
'accuracy': accuracy
}, self)
return loss
def __call__(self, x, t):
h = F.relu(self['conv1/7x7_s2'](x))
h = F.local_response_normalization(
F.max_pooling_2d(h, 3, stride=2), n=5, alpha=(1e-4)/5, k=1)
h = F.relu(self['conv2/3x3_reduce'](h))
h = F.relu(self['conv2/3x3'](h))
h = F.max_pooling_2d(F.local_response_normalization(
h, n=5, alpha=(1e-4)/5, k=1), 3, stride=2)
h = self.call_inception(h, 'inception_3a')
h = self.call_inception(h, 'inception_3b')
h = F.max_pooling_2d(h, 3, stride=2)
h = self.call_inception(h, 'inception_4a')
l = F.average_pooling_2d(h, 5, stride=3)
l = F.relu(self['loss1/conv'](l))
l = F.dropout(F.relu(self['loss1/fc'](l)), 0.7, train=self.train)
l = self['loss1/classifier'](l)
loss1 = F.softmax_cross_entropy(l, t)
h = self.call_inception(h, 'inception_4b')
h = self.call_inception(h, 'inception_4c')
h = self.call_inception(h, 'inception_4d')
l = F.average_pooling_2d(h, 5, stride=3)
l = F.relu(self['loss2/conv'](l))
l = F.dropout(F.relu(self['loss2/fc'](l)), 0.7, train=self.train)
l = self['loss2/classifier'](l)
loss2 = F.softmax_cross_entropy(l, t)
h = self.call_inception(h, 'inception_4e')
h = F.max_pooling_2d(h, 3, stride=2)
h = self.call_inception(h, 'inception_5a')
h = self.call_inception(h, 'inception_5b')
h = F.average_pooling_2d(h, 7, stride=1)
h = self['loss3/classifier'](F.dropout(h, 0.4, train=self.train))
loss3 = F.softmax_cross_entropy(h, t)
loss = 0.3 * (loss1 + loss2) + loss3
accuracy = F.accuracy(h, t)
chainer.report({
'loss': loss,
'loss1': loss1,
'loss2': loss2,
'loss3': loss3,
'accuracy': accuracy
}, self)
return loss
def __call__(self, x, t):
h = F.max_pooling_2d(F.local_response_normalization(
F.relu(self.conv1(x))), 3, stride=2)
h = F.max_pooling_2d(F.local_response_normalization(
F.relu(self.conv2(h))), 3, stride=2)
h = F.relu(self.conv3(h))
h = F.relu(self.conv4(h))
h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
h = F.dropout(F.relu(self.fc6(h)))
h = F.dropout(F.relu(self.fc7(h)))
h = self.fc8(h)
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def lf(x):
mu, ln_var = self.encode(x)
batchsize = len(mu.data)
# reconstruction loss
rec_loss = 0
for l in six.moves.range(k):
z = F.gaussian(mu, ln_var)
rec_loss += F.bernoulli_nll(x, self.decode(z, sigmoid=False)) \
/ (k * batchsize)
self.rec_loss = rec_loss
self.loss = self.rec_loss + \
C * gaussian_kl_divergence(mu, ln_var) / batchsize
chainer.report(
{'rec_loss': rec_loss, 'loss': self.loss}, observer=self)
return self.loss
def __call__(self, x, t):
test = not self.train
finetune = self.finetune
h = self.call_conv_bn_sc(x, 'conv1/7x7_s2', test=test, finetune=finetune)
h = F.max_pooling_2d(h, 3, stride=2, pad=1)
h = self.call_conv_bn_sc(h, 'conv2/3x3_reduce', test=test, finetune=finetune)
h = self.call_conv_bn_sc(h, 'conv2/3x3', test=test)
h = F.max_pooling_2d(h, 3, stride=2, pad=1)
h = self.call_inception_bn(h, 'inception_3a', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_3b', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_3c', test=test, finetune=finetune)
a = F.average_pooling_2d(h, 5, stride=3)
a = self.call_conv_bn_sc(a, 'loss1/conv', test=test, finetune=finetune)
a = self.call_fc_bn_sc(a, 'loss1/fc', test=test, finetune=finetune)
a = self['loss1/classifier'](a)
loss1 = F.softmax_cross_entropy(a, t)
h = self.call_inception_bn(h, 'inception_4a', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_4b', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_4c', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_4d', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_4e', test=test, finetune=finetune)
b = F.average_pooling_2d(h, 5, stride=3)
b = self.call_conv_bn_sc(b, 'loss2/conv', test=test, finetune=finetune)
b = self.call_fc_bn_sc(b, 'loss2/fc', test=test, finetune=finetune)
b = self['loss2/classifier'](b)
loss2 = F.softmax_cross_entropy(b, t)
h = self.call_inception_bn(h, 'inception_5a', test=test, finetune=finetune)
h = self.call_inception_bn(h, 'inception_5b', test=test, finetune=finetune)
h = F.average_pooling_2d(h, 7, stride=1)
h = self['loss3/classifier'](h)
loss3 = F.softmax_cross_entropy(h, t)
loss = 0.3 * (loss1 + loss2) + loss3
accuracy = F.accuracy(h, t)
chainer.report({
'loss': loss,
'loss1': loss1,
'loss2': loss2,
'loss3': loss3,
'accuracy': accuracy
}, self)
return loss
def __call__(self, x, t):
test = not self.train
h = F.max_pooling_2d(
F.relu(self.norm1(self.conv1(x), test=test)), 3, stride=2, pad=1)
h = F.max_pooling_2d(
F.relu(self.norm2(self.conv2(h), test=test)), 3, stride=2, pad=1)
h = self.inc3a(h)
h = self.inc3b(h)
h = self.inc3c(h)
h = self.inc4a(h)
a = F.average_pooling_2d(h, 5, stride=3)
a = F.relu(self.norma(self.conva(a), test=test))
a = F.relu(self.norma2(self.lina(a), test=test))
a = self.outa(a)
loss1 = F.softmax_cross_entropy(a, t)
h = self.inc4b(h)
h = self.inc4c(h)
h = self.inc4d(h)
b = F.average_pooling_2d(h, 5, stride=3)
b = F.relu(self.normb(self.convb(b), test=test))
b = F.relu(self.normb2(self.linb(b), test=test))
b = self.outb(b)
loss2 = F.softmax_cross_entropy(b, t)
h = self.inc4e(h)
h = self.inc5a(h)
h = F.average_pooling_2d(self.inc5b(h), 7)
h = self.out(h)
loss3 = F.softmax_cross_entropy(h, t)
loss = 0.3 * (loss1 + loss2) + loss3
accuracy = F.accuracy(h, t)
chainer.report({
'loss': loss,
'loss1': loss1,
'loss2': loss2,
'loss3': loss3,
'accuracy': accuracy,
}, self)
return loss
def __call__(self, trainer):
with chainer.no_backprop_mode():
references = []
hypotheses = []
for i in range(0, len(self.test_data), self.batch):
sources, targets = zip(*self.test_data[i:i + self.batch])
references.extend([[t.tolist()] for t in targets])
sources = [
chainer.dataset.to_device(self.device, x) for x in sources]
ys = [y.tolist()
for y in self.model.translate(sources, self.max_length)]
hypotheses.extend(ys)
bleu = bleu_score.corpus_bleu(
references, hypotheses,
smoothing_function=bleu_score.SmoothingFunction().method1)
chainer.report({self.key: bleu})
def __call__(self, x, t):
h = self.bn1(self.conv1(x), test=not self.train)
h = F.max_pooling_2d(F.relu(h), 3, stride=2)
h = self.bn2(self.conv2(h), test=not self.train)
h = F.max_pooling_2d(F.relu(h), 3, stride=2)
h = F.relu(self.conv3(h))
h = F.relu(self.conv4(h))
h = F.max_pooling_2d(F.relu(self.conv5(h)), 3, stride=2)
h = F.dropout(F.relu(self.fc6(h)), train=self.train)
h = F.dropout(F.relu(self.fc7(h)), train=self.train)
h = self.fc8(h)
#self.loss = F.softmax_cross_entropy(h, t)
#self.accuracy = F.accuracy(h, t)
#return self.loss
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def __call__(self, x, t):
h = self.conv1_2(F.relu(self.conv1_1(x)))
h = F.max_pooling_2d(F.relu(h), 2, stride=2)
h = self.conv2_2(F.relu(self.conv2_1(h)))
h = F.max_pooling_2d(F.relu(h), 2, stride=2)
h = self.conv3_3(F.relu(self.conv3_2(F.relu(self.conv3_1(h)))))
h = F.max_pooling_2d(F.relu(h), 2, stride=2)
h = self.conv4_3(F.relu(self.conv4_2(F.relu(self.conv4_1(h)))))
h = F.max_pooling_2d(F.relu(h), 2, stride=2)
h = self.conv5_3(F.relu(self.conv5_2(F.relu(self.conv5_1(h)))))
h = F.max_pooling_2d(F.relu(h), 2, stride=2)
h = F.dropout(F.relu(self.fc6(h)), train=self.train, ratio=0.5)
h = F.dropout(F.relu(self.fc7(h)), train=self.train, ratio=0.5)
h = self.fc8(h)
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def _loss_predictor(self, predictor, output, target, d_fake):
b, _, t = d_fake.data.shape
loss_mse = (F.mean_absolute_error(output, target))
chainer.report({'mse': loss_mse}, predictor)
loss_adv = F.sum(F.softplus(-d_fake)) / (b * t)
chainer.report({'adversarial': loss_adv}, predictor)
loss = self.loss_config.mse * loss_mse + self.loss_config.adversarial * loss_adv
chainer.report({'loss': loss}, predictor)
return loss
def __call__(self, x, t=None):
# conv1
h = F.relu(self.conv1_1(x))
conv1_1 = h
h = F.relu(self.conv1_2(conv1_1))
conv1_2 = h
h = F.max_pooling_2d(conv1_2, 2, stride=2, pad=0)
pool1 = h # 1/2
# conv2
h = F.relu(self.conv2_1(pool1))
conv2_1 = h
h = F.relu(self.conv2_2(conv2_1))
conv2_2 = h
h = F.max_pooling_2d(conv2_2, 2, stride=2, pad=0)
pool2 = h # 1/4
# conv3
h = F.relu(self.conv3_1(pool2))
conv3_1 = h
h = F.relu(self.conv3_2(conv3_1))
conv3_2 = h
h = F.relu(self.conv3_3(conv3_2))
conv3_3 = h
h = F.max_pooling_2d(conv3_3, 2, stride=2, pad=0)
pool3 = h # 1/8
# conv4
h = F.relu(self.conv4_1(pool3))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
h = F.max_pooling_2d(h, 2, stride=2, pad=0)
pool4 = h # 1/16
# conv5
h = F.relu(self.conv5_1(pool4))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = F.max_pooling_2d(h, 2, stride=2, pad=0)
pool5 = h # 1/32
# fc6
h = F.relu(self.fc6(pool5))
h = F.dropout(h, ratio=.5)
fc6 = h # 1/32
# fc7
h = F.relu(self.fc7(fc6))
h = F.dropout(h, ratio=.5)
fc7 = h # 1/32
# score_fr
h = self.score_fr(fc7)
score_fr = h # 1/32
# upscore2
h = self.upscore2(score_fr)
upscore2 = h # 1/16
# score_pool4
h = self.score_pool4(pool4)
score_pool4 = h # 1/16
# score_pool4c
h = score_pool4[:, :,
5:5 + upscore2.shape[2],
5:5 + upscore2.shape[3]]
score_pool4c = h # 1/16
# fuse_pool4
h = upscore2 + score_pool4c
fuse_pool4 = h # 1/16
# upscore16
h = self.upscore16(fuse_pool4)
upscore16 = h # 1/1
# score
h = upscore16[:, :, 27:27 + x.shape[2], 27:27 + x.shape[3]]
score = h # 1/1
self.score = score
if t is None:
assert not chainer.configuration.config.train
return
loss = F.softmax_cross_entropy(self.score, t, normalize=False)
if np.isnan(float(loss.data)):
raise ValueError('Loss value is nan.')
chainer.report({'loss': loss}, self)
return loss
def __call__(self, tgt_img, src_imgs, intrinsics, inv_intrinsics):
"""
Args:
tgt_img: target image. Shape is (Batch, 3, H, W)
src_imgs: source images. Shape is (Batch, ?, 3, H, W)
intrinsics: Shape is (Batch, ?, 3, 3)
Return:
loss (Variable).
"""
batchsize, n_sources, _, H, W = src_imgs.shape
stacked_src_imgs = self.xp.reshape(src_imgs, (batchsize, -1, H, W))
pred_disps = self.disp_net(tgt_img)
pred_depthes = [1. / d for d in pred_disps]
do_exp = self.exp_reg is not None and self.exp_reg > 0
pred_poses, pred_maskes = self.pose_net(tgt_img,
stacked_src_imgs,
do_exp=do_exp)
smooth_loss, exp_loss, pixel_loss = 0, 0, 0
ssim_loss = 0
n_scales = len(pred_depthes)
start, stop = create_timer()
sum_time = 0
for ns in range(n_scales):
curr_img_size = (H // (2**ns), W // (2**ns))
curr_tgt_img = F.resize_images(tgt_img, curr_img_size).data
curr_src_imgs = F.resize_images(stacked_src_imgs,
curr_img_size).data
# Smoothness regularization
if self.smooth_reg:
smooth_loss += (self.smooth_reg / (2 ** ns)) * \
self.compute_smooth_loss(pred_disps[ns])
# smooth_loss += (self.smooth_reg / (2 ** ns)) * \
# self.compute_disp_smooth(curr_tgt_img,
# pred_disps[ns])
curr_pred_depthes = pred_depthes[ns]
curr_pred_depthes = F.reshape(curr_pred_depthes,
(batchsize, 1, -1))
curr_pred_depthes = F.broadcast_to(
curr_pred_depthes, (batchsize, 3, curr_pred_depthes.shape[2]))
curr_intrinsics = intrinsics[:, ns]
if self.exp_reg:
curr_pred_mask = pred_maskes[ns]
for i in range(n_sources):
# Inverse warp the source image to the target image frame
curr_proj_img = projective_inverse_warp(
curr_src_imgs[:, i * 3:(i + 1) * 3], curr_pred_depthes,
pred_poses[i], curr_intrinsics)
curr_proj_error = F.absolute(curr_proj_img - curr_tgt_img)
mask = (curr_proj_img.data == 0).prod(
1, keepdims=True).astype('bool')
mask = self.xp.broadcast_to(mask, curr_proj_error.shape)
curr_proj_error = F.where(
mask, self.xp.zeros(curr_proj_error.shape, dtype='f'),
curr_proj_error)
# curr_proj_error *= (1 - (curr_proj_img.data == 0).prod(1, keepdims=True))
# explainability regularization
if self.exp_reg:
pred_exp_logits = curr_pred_mask[:, i:i + 1, :, :]
exp_loss += self.exp_reg * \
self.compute_exp_reg_loss(pred_exp_logits)
pred_exp = F.sigmoid(pred_exp_logits)
pred_exp = F.broadcast_to(
pred_exp,
(batchsize, 3, curr_img_size[0], curr_img_size[1]))
pixel_loss += F.mean(curr_proj_error * pred_exp)
else:
pixel_loss += F.mean(curr_proj_error)
if self.ssim_rate:
ssim_error = self.compute_ssim(curr_proj_img,
curr_tgt_img)
ssim_error *= (1 - mask)
ssim_loss += F.mean(ssim_error)
total_loss = (1 - self.ssim_rate) * pixel_loss + self.ssim_rate * ssim_loss + \
smooth_loss + exp_loss
chainer.report({'total_loss': total_loss}, self)
chainer.report({'pixel_loss': pixel_loss}, self)
chainer.report({'smooth_loss': smooth_loss}, self)
chainer.report({'exp_loss': exp_loss}, self)
chainer.report({'ssim_loss': ssim_loss}, self)
return total_loss
def segment(self, x, t=None):
# conv1
self.conv1_1.pad = (100, 100)
h = F.relu(self.conv1_1(x))
conv1_1 = h
h = F.relu(self.conv1_2(conv1_1))
conv1_2 = h
h = _max_pooling_2d(conv1_2)
pool1 = h # 1/2
# conv2
h = F.relu(self.conv2_1(pool1))
conv2_1 = h
h = F.relu(self.conv2_2(conv2_1))
conv2_2 = h
h = _max_pooling_2d(conv2_2)
pool2 = h # 1/4
# conv3
h = F.relu(self.conv3_1(pool2))
conv3_1 = h
h = F.relu(self.conv3_2(conv3_1))
conv3_2 = h
h = F.relu(self.conv3_3(conv3_2))
conv3_3 = h
h = _max_pooling_2d(conv3_3)
pool3 = h # 1/8
# conv4
h = F.relu(self.conv4_1(pool3))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
h = _max_pooling_2d(h)
pool4 = h # 1/16
# conv5
h = F.relu(self.conv5_1(pool4))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = _max_pooling_2d(h)
pool5 = h # 1/32
# fc6
h = F.relu(self.fc6(pool5))
h = F.dropout(h, ratio=.5)
fc6 = h # 1/32
# fc7
h = F.relu(self.fc7(fc6))
h = F.dropout(h, ratio=.5)
fc7 = h # 1/32
# score_fr
h = self.score_fr(fc7)
score_fr = h # 1/32
# score_pool3
h = self.score_pool3(pool3)
score_pool3 = h # 1/8
# score_pool4
h = self.score_pool4(pool4)
score_pool4 = h # 1/16
# upscore2
h = self.upscore2(score_fr)
upscore2 = h # 1/16
# score_pool4c
h = score_pool4[:, :, 5:5 + upscore2.shape[2], 5:5 + upscore2.shape[3]]
score_pool4c = h # 1/16
# fuse_pool4
h = upscore2 + score_pool4c
fuse_pool4 = h # 1/16
# upscore_pool4
h = self.upscore_pool4(fuse_pool4)
upscore_pool4 = h # 1/8
# score_pool4c
h = score_pool3[:, :, 9:9 + upscore_pool4.shape[2],
9:9 + upscore_pool4.shape[3]]
score_pool3c = h # 1/8
# fuse_pool3
h = upscore_pool4 + score_pool3c
fuse_pool3 = h # 1/8
# upscore8
h = self.upscore8(fuse_pool3)
upscore8 = h # 1/1
# score
h = upscore8[:, :, 31:31 + x.shape[2], 31:31 + x.shape[3]]
score = h # 1/1
self.score = score
if t is None:
assert not chainer.config.train
return
loss = F.softmax_cross_entropy(score, t, normalize=True)
if np.isnan(float(loss.data)):
raise ValueError('Loss is nan.')
chainer.report({'loss': loss}, self)
self.conv1_1.pad = (1, 1)
return loss
def update_core(self):
gen_optimizer = self.get_optimizer('gen')
dis_optimizer = self.get_optimizer('dis')
gen, dis = self.gen, self.dis
## image conversion
batch = self.get_iterator('main').next()
x_in, t_out = self.converter(batch, self.device)
x_in = Variable(x_in)
t_out = Variable(t_out)
x_out = gen(add_noise(x_in, sigma=self.args.noise))
x_in_out_copy = Variable(
self._buffer.query(F.concat([x_in, x_out]).data))
loss_gen = 0
# reconstruction error
if self.args.lambda_rec_l1 > 0:
# loss_rec_l1 = F.mean_absolute_error(x_out, t_out)
loss_rec_l1 = self.loss_avg(x_out,
t_out,
ksize=self.args.loss_ksize,
norm='l1')
loss_gen = loss_gen + self.args.lambda_rec_l1 * loss_rec_l1
chainer.report({'loss_L1': loss_rec_l1}, gen)
if self.args.lambda_rec_l2 > 0:
# loss_rec_l2 = F.mean_squared_error(x_out, t_out)
loss_rec_l2 = self.loss_avg(x_out,
t_out,
ksize=self.args.loss_ksize,
norm='l2')
loss_gen = loss_gen + self.args.lambda_rec_l2 * loss_rec_l2
chainer.report({'loss_L2': loss_rec_l2}, gen)
# total variation
if self.args.lambda_tv > 0:
loss_tv = self.total_variation2(x_out, self.args.tv_tau)
loss_gen = loss_gen + self.args.lambda_tv * loss_tv
chainer.report({'loss_tv': loss_tv}, gen)
# Adversarial loss
if self.args.lambda_dis > 0:
y_fake = dis(F.concat([x_in, x_out]))
#batchsize,_,w,h = y_fake.data.shape
#loss_dis = F.sum(F.softplus(-y_fake)) / batchsize / w / h
loss_adv = self.loss_func_comp(y_fake, 1.0, self.args.dis_jitter)
chainer.report({'loss_dis': loss_adv}, gen)
loss_gen = loss_gen + self.args.lambda_dis * loss_adv
# update generator model
gen.cleargrads()
loss_gen.backward()
gen_optimizer.update(loss=loss_gen)
## discriminator
if self.args.lambda_dis > 0:
loss_real = self.loss_func_comp(dis(F.concat([x_in, t_out])), 1.0,
self.args.dis_jitter)
loss_fake = self.loss_func_comp(dis(x_in_out_copy), 0.0,
self.args.dis_jitter)
chainer.report({'loss_fake': loss_fake}, dis)
chainer.report({'loss_real': loss_real}, dis)
## mis-matched input-output pair should be discriminated as fake
if self._buffer.num_imgs > 40 and self.args.lambda_mispair > 0:
f_in = self.gen.xp.concatenate(
random.sample(self._buffer.images, len(x_in)))
f_in = Variable(f_in[:, :x_in.shape[1], :, :])
loss_mispair = self.loss_func_comp(
dis(F.concat([f_in, t_out])), 0.0, self.args.dis_jitter)
chainer.report({'loss_mispair': loss_mispair}, dis)
else:
loss_mispair = 0
loss_dis = 0.5 * (loss_fake + loss_real
) + self.args.lambda_mispair * loss_mispair
dis.cleargrads()
loss_dis.backward()
dis_optimizer.update(loss=loss_dis)
def __call__(self, x_a, x_p, x_n):
y_a, y_p, y_n = (self.predictor(x) for x in (x_a, x_p, x_n))
loss = F.triplet(y_a, y_p, y_n)
report({'loss': loss}, self)
return loss
def test_report_with_unregistered_observer(self):
reporter = chainer.Reporter()
observer = object()
with reporter:
with self.assertRaises(KeyError):
chainer.report({'x': 1}, observer)
def __call__(self, x, t):
y = self.lin(x)
loss = F.softmax_cross_entropy(y, t)
acc = F.accuracy(y, t)
chainer.report({'loss': loss, 'acc': acc}, self)
return loss
def __call__(self, x, img_real):
if type(img_real) != chainer.variable.Variable: # if validation set
img_real = Variable(img_real)
## Compute latent space from BOLD
z = self.predictor(x)
## Generate images from latent space
img_fake = self.pretrained_gan.generate_img_from_z(z)
img_fake = F.clip(img_fake, -1.0, 1.0) # avoid slight overflow of values (after tanh, up to 1.07)
img_fake.volatile = 'OFF' ; img_real.volatile = 'OFF' # workaround an issue during validation
## Get activations of perceptual features
_, layer_activations_fake = self.featnet( img_fake, train=False, return_activations=True )
_, layer_activations_real = self.featnet( img_real, train=False, return_activations=True )
# Note that featnet can also return the non-softmaxed final layer activations (=the classes, here in _ ).
# Got some bizarre (and no better) results for natural images when also including a class-matching loss.
# But (as mentioned in the paper): A loss on higher layers of a convnet trained with a discrete set of
# classes (such as ImageNet classes) may *restrict* your reconstructions to these classes, which is
# not desired. Computing a loss within a continuous semantic space may be a solution here.
## Compute perceptual losses
loss = 0.0
if self.featnet != None:
for layer_idx in ['pixel'] + args.featn_layers:
if layer_idx == 'pixel':
# compute pixel loss l_px
loss_px = args.lambda_pixel * (
F.mean_absolute_error( F.resize_images(img_fake,
(args.small_img_dims,args.small_img_dims)),
F.resize_images(img_real,
(args.small_img_dims,args.small_img_dims)) ) )
loss += loss_px
else:
layer_idx = int(layer_idx)
activ_fake_pos = F.hard_sigmoid( layer_activations_fake[layer_idx]*3.0 - 3.*args.featthre )
activ_real_pos = F.hard_sigmoid( layer_activations_real[layer_idx]*3.0 - 3.*args.featthre )
# using hard_sigmoid for a differentiable binarization at threshold 1.0
if int(layer_idx) == 0: # negative feature activations only make sense for conv1
activ_fake_neg = F.hard_sigmoid( -1.0*layer_activations_fake[layer_idx]*3.0 - 3.*args.featthre )
activ_real_neg = F.hard_sigmoid( -1.0*layer_activations_real[layer_idx]*3.0 - 3.*args.featthre )
mask_real = (activ_real_pos.data + activ_real_neg.data) > 0
else: # only use positive activations
mask_real = activ_real_pos.data > 0
loss_pr_neg = 0
if np.sum(mask_real[:]) > 0.0: # if there are any activations above 1.0
# compute l_l,m
loss_mag = args.lambda_magnitude * (
F.mean_squared_error(layer_activations_fake[layer_idx][mask_real],
layer_activations_real[layer_idx][mask_real]) )
else: # warn and set magnitude loss to 0.0 (does not happen)
loss_mag = 0.0
print "Warning: No magnitude loss"
loss += loss_mag
report({'loss': loss}, self)
# Use this code to check whether gradients were computed:
#self.predictor.l1.cleargrads()
#loss.backward()
#print "Gradients: ", self.predictor.l1.W.grad
# Do this for all new loss terms.
return loss
def __call__(self, x):
self.args.append(x)
chainer.report({'loss': x.sum()}, self)
def loss(
self,
class_id,
quaternion_true,
translation_true,
quaternion_pred,
translation_pred,
confidence_pred,
pitch=None,
origin=None,
grid_target=None,
grid_nontarget_empty=None,
):
xp = self.xp
B = class_id.shape[0]
# prepare
quaternion_true = quaternion_true.astype(np.float32)
translation_true = translation_true.astype(np.float32)
if pitch is not None:
pitch = pitch.astype(np.float32)
if origin is not None:
origin = origin.astype(np.float32)
if grid_target is not None:
grid_target = grid_target.astype(np.float32)
if grid_nontarget_empty is not None:
grid_nontarget_empty = grid_nontarget_empty.astype(np.float32)
loss = 0
for i in range(B):
T_cad2cam_pred = morefusion.functions.transformation_matrix(
quaternion_pred[i], translation_pred[i]
) # (P, 4, 4)
T_cad2cam_true = morefusion.functions.transformation_matrix(
quaternion_true[i], translation_true[i]
) # (4, 4)
class_id_i = int(class_id[i])
cad_pcd = self._models.get_pcd(class_id=class_id_i)
cad_pcd = cad_pcd[np.random.permutation(cad_pcd.shape[0])[:500]]
cad_pcd = xp.asarray(cad_pcd, dtype=np.float32)
is_symmetric = (
class_id_i in morefusion.datasets.ycb_video.class_ids_symmetric
)
if self._loss in ["add", "add+occupancy"]:
is_symmetric = False
elif self._loss in ["add_s"]:
is_symmetric = True
else:
assert self._loss in ["add/add_s", "add/add_s+occupancy"]
add = morefusion.functions.average_distance(
cad_pcd,
T_cad2cam_true,
T_cad2cam_pred,
symmetric=is_symmetric,
)
del cad_pcd, T_cad2cam_true, is_symmetric
keep = confidence_pred[i].array > 0
loss_i = F.mean(
add[keep] * confidence_pred[i][keep]
- self._lambda_confidence * F.log(confidence_pred[i][keep])
)
if self._loss in ["add+occupancy", "add/add_s+occupancy"]:
solid_pcd = self._models.get_solid_voxel_grid(
class_id=class_id_i
)
solid_pcd = xp.asarray(solid_pcd.points, dtype=np.float32)
kwargs = dict(
pitch=float(pitch[i]),
origin=cuda.to_cpu(origin[i]),
dims=(self._voxel_dim,) * 3,
threshold=2.0,
)
grid_target_pred = morefusion.functions.pseudo_occupancy_voxelization( # NOQA
points=morefusion.functions.transform_points(
solid_pcd, T_cad2cam_pred[i]
),
**kwargs,
)
# give reward intersection w/ target voxels
intersection = F.sum(grid_target_pred * grid_target[i])
denominator = F.sum(grid_target[i]) + 1e-16
loss_i -= (
self._loss_scale["occupancy"] * intersection / denominator
)
# penalize intersection w/ nontarget voxels
intersection = F.sum(
grid_target_pred * grid_nontarget_empty[i]
)
denominator = F.sum(grid_target_pred) + 1e-16
loss_i += (
self._loss_scale["occupancy"] * intersection / denominator
)
loss += loss_i
loss /= B
values = {"loss": loss}
chainer.report(values, observer=self)
return loss
def update_core(self):
opt_g = self.get_optimizer('opt_g')
opt_f = self.get_optimizer('opt_f')
opt_x = self.get_optimizer('opt_x')
opt_y = self.get_optimizer('opt_y')
self._iter += 1
# learning rate decay
if self.is_new_epoch and self.epoch >= self.args.lrdecay_start:
noise_decay = self.args.noise / self.args.lrdecay_period
decay_step = self.init_alpha / self.args.lrdecay_period
print('lr decay', decay_step)
if opt_g.alpha > decay_step:
opt_g.alpha -= decay_step
if opt_f.alpha > decay_step:
opt_f.alpha -= decay_step
if opt_x.alpha > decay_step:
opt_x.alpha -= decay_step
if opt_y.alpha > decay_step:
opt_y.alpha -= decay_step
self.args.noise -= noise_decay
# get mini-batch
batch_x = self.get_iterator('main').next()
batch_y = self.get_iterator('train_B').next()
x = Variable(self.converter(batch_x, self.device))
y = Variable(self.converter(batch_y, self.device))
### generator
# X => Y => X
x_y = self.gen_g(losses.add_noise(
x, sigma=self.args.noise)) # noise injection
if self.args.conditional_discriminator:
x_y_copy = Variable(self._buffer_y.query(F.concat([x_y, x]).data))
else:
x_y_copy = Variable(self._buffer_y.query(x_y.data))
x_y_x = self.gen_f(x_y)
loss_cycle_x = losses.loss_avg(x_y_x,
x,
ksize=self.args.cycle_ksize,
norm='l1')
loss_gen_g_adv = 0
if self.args.gen_start < self._iter:
if self.args.conditional_discriminator:
if self.args.wgan:
loss_gen_g_adv = -F.average(self.dis_y(F.concat([x_y, x])))
else:
loss_gen_g_adv = losses.loss_func_comp(
self.dis_y(F.concat([x_y, x])), 1.0)
else:
if self.args.wgan:
loss_gen_g_adv = -F.average(self.dis_y(x_y))
else:
loss_gen_g_adv = losses.loss_func_comp(
self.dis_y(x_y), 1.0)
# Y => X => Y
loss_gen_f_adv = 0
y_x = self.gen_f(losses.add_noise(
y, sigma=self.args.noise)) # noise injection
if self.args.conditional_discriminator:
y_x_copy = Variable(self._buffer_x.query(F.concat([y_x, y]).data))
else:
y_x_copy = Variable(self._buffer_x.query(y_x.data))
y_x_y = self.gen_g(y_x)
loss_cycle_y = losses.loss_avg(y_x_y,
y,
ksize=self.args.cycle_ksize,
norm='l1')
if self.args.gen_start < self._iter:
if self.args.conditional_discriminator:
if self.args.wgan:
loss_gen_f_adv = -F.average(self.dis_x(F.concat([y_x, y])))
else:
loss_gen_f_adv = losses.loss_func_comp(
self.dis_x(F.concat([y_x, y])), 1.0)
else:
if self.args.wgan:
loss_gen_f_adv = -F.average(self.dis_x(y_x))
else:
loss_gen_f_adv = losses.loss_func_comp(
self.dis_x(y_x), 1.0)
## total loss for generators
loss_gen = (self.args.lambda_dis_y * loss_gen_g_adv +
self.args.lambda_dis_x * loss_gen_f_adv) + (
self.args.lambda_A * loss_cycle_x +
self.args.lambda_B * loss_cycle_y)
## idempotence: f shouldn't change x
if self.args.lambda_idempotence > 0:
loss_idem_x = F.mean_absolute_error(y_x, self.gen_f(y_x))
loss_idem_y = F.mean_absolute_error(x_y, self.gen_g(x_y))
loss_gen = loss_gen + self.args.lambda_idempotence * (loss_idem_x +
loss_idem_y)
if self.report_start < self._iter:
chainer.report({'loss_idem': loss_idem_x}, self.gen_f)
chainer.report({'loss_idem': loss_idem_y}, self.gen_g)
if self.args.lambda_domain > 0:
loss_dom_x = F.mean_absolute_error(x, self.gen_f(x))
loss_dom_y = F.mean_absolute_error(y, self.gen_g(y))
if self._iter < self.args.warmup:
loss_gen = loss_gen + max(self.args.lambda_domain,
1.0) * (loss_dom_x + loss_dom_y)
else:
loss_gen = loss_gen + self.args.lambda_domain * (loss_dom_x +
loss_dom_y)
if self.report_start < self._iter:
chainer.report({'loss_dom': loss_dom_x}, self.gen_f)
chainer.report({'loss_dom': loss_dom_y}, self.gen_g)
## images before/after conversion should look similar
if self.args.lambda_identity_x > 0 or self._iter < self.args.warmup:
loss_id_x = losses.loss_avg(x,
x_y,
ksize=self.args.id_ksize,
norm='l2')
if self._iter < self.args.warmup:
loss_gen = loss_gen + max(self.args.lambda_identity_x,
2.0) * loss_id_x
else:
loss_gen = loss_gen + self.args.lambda_identity_x * loss_id_x
if self.args.lambda_identity_x > 0 and self.report_start < self._iter:
chainer.report({'loss_id': loss_id_x}, self.gen_g)
if self.args.lambda_identity_y > 0 or self._iter < self.args.warmup:
loss_id_y = losses.loss_avg(y,
y_x,
ksize=self.args.id_ksize,
norm='l2')
if self._iter < self.args.warmup:
loss_gen = loss_gen + max(self.args.lambda_identity_y,
2.0) * loss_id_y
else:
loss_gen = loss_gen + self.args.lambda_identity_y * loss_id_y
if self.args.lambda_identity_y > 0 and self.report_start < self._iter:
chainer.report({'loss_id': loss_id_y}, self.gen_f)
if self.args.lambda_grad > 0:
loss_grad_x = losses.loss_grad(x, x_y, self.args.grad_norm)
loss_gen = loss_gen + self.args.lambda_grad * loss_grad_x
#loss_grad_y = losses.loss_grad(y,y_x)
#loss_gen = loss_gen + self.args.lambda_grad * loss_grad_y
if self.report_start < self._iter:
chainer.report({'loss_grad': loss_grad_x}, self.gen_g)
#chainer.report({'loss_grad': loss_grad_y}, self.gen_f)
## background should be preserved
if self.args.lambda_air > 0:
loss_air_x = self.args.lambda_air * losses.loss_range_comp(
x, x_y, 0.9, norm='l1')
loss_air_y = self.args.lambda_air * losses.loss_range_comp(
y, y_x, 0.9, norm='l1')
loss_gen = loss_gen + (loss_air_x + loss_air_y)
if self.report_start < self._iter:
chainer.report({'loss_air': 0.1 * loss_air_x}, self.gen_g)
chainer.report({'loss_air': 0.1 * loss_air_y}, self.gen_f)
## total variation
if self.args.lambda_tv > 0:
loss_tv = losses.total_variation(x_y, self.args.tv_tau)
loss_gen = loss_gen + self.args.lambda_tv * loss_tv
if self.report_start < self._iter:
chainer.report({'loss_tv': loss_tv}, self.gen_g)
if self.report_start < self._iter:
chainer.report({'loss_cycle_x': loss_cycle_x}, self.gen_f)
chainer.report({'loss_adv': loss_gen_g_adv}, self.gen_g)
chainer.report({'loss_cycle_y': loss_cycle_y}, self.gen_g)
chainer.report({'loss_adv': loss_gen_f_adv}, self.gen_f)
self.gen_f.cleargrads()
self.gen_g.cleargrads()
loss_gen.backward()
opt_f.update()
opt_g.update()
### discriminator
for t in range(self.args.n_critics):
if self.args.wgan: ## synthesised -, real +
loss_dis_y_fake = F.average(self.dis_y(x_y_copy))
eps = self.xp.random.uniform(0, 1, size=len(batch_y)).astype(
self.xp.float32)[:, None, None, None]
if self.args.conditional_discriminator:
loss_dis_y_real = -F.average(self.dis_y(F.concat([y, x])))
y_mid = eps * F.concat([y, x]) + (1.0 - eps) * x_y_copy
else:
loss_dis_y_real = -F.average(self.dis_y(y))
y_mid = eps * y + (1.0 - eps) * x_y_copy
# gradient penalty
gd_y, = chainer.grad([self.dis_y(y_mid)], [y_mid],
enable_double_backprop=True)
gd_y = F.sqrt(F.batch_l2_norm_squared(gd_y))
loss_dis_y_gp = F.mean_squared_error(
gd_y, self.xp.ones_like(gd_y.data))
if self.report_start < self._iter:
chainer.report({'loss_real': loss_dis_y_real}, self.dis_y)
chainer.report({'loss_fake': loss_dis_y_fake}, self.dis_y)
chainer.report(
{'loss_gp': self.args.lambda_wgan_gp * loss_dis_y_gp},
self.dis_y)
loss_dis_y = (loss_dis_y_real + loss_dis_y_fake +
self.args.lambda_wgan_gp * loss_dis_y_gp)
self.dis_y.cleargrads()
loss_dis_y.backward()
opt_y.update()
## discriminator for B=>A
loss_dis_x_fake = F.average(self.dis_x(y_x_copy))
if self.args.conditional_discriminator:
loss_dis_x_real = -F.average(self.dis_x(F.concat([x, y])))
x_mid = eps * F.concat([x, y]) + (1.0 - eps) * y_x_copy
else:
loss_dis_x_real = -F.average(self.dis_x(x))
x_mid = eps * x + (1.0 - eps) * y_x_copy
# gradient penalty
gd_x, = chainer.grad([self.dis_x(x_mid)], [x_mid],
enable_double_backprop=True)
gd_x = F.sqrt(F.batch_l2_norm_squared(gd_x))
loss_dis_x_gp = F.mean_squared_error(
gd_x, self.xp.ones_like(gd_x.data))
if self.report_start < self._iter:
chainer.report({'loss_real': loss_dis_x_real}, self.dis_x)
chainer.report({'loss_fake': loss_dis_x_fake}, self.dis_x)
chainer.report(
{'loss_gp': self.args.lambda_wgan_gp * loss_dis_x_gp},
self.dis_x)
loss_dis_x = (loss_dis_x_real + loss_dis_x_fake +
self.args.lambda_wgan_gp * loss_dis_x_gp)
self.dis_x.cleargrads()
loss_dis_x.backward()
opt_x.update()
else:
## discriminator for A=>B (real:1, fake:0)
loss_dis_y_fake = losses.loss_func_comp(
self.dis_y(x_y_copy), 0.0, self.args.dis_jitter)
if self.args.conditional_discriminator:
loss_dis_y_real = losses.loss_func_comp(
self.dis_y(F.concat([y, x])), 1.0,
self.args.dis_jitter)
else:
loss_dis_y_real = losses.loss_func_comp(
self.dis_y(y), 1.0, self.args.dis_jitter)
loss_dis_y = (loss_dis_y_fake + loss_dis_y_real) * 0.5
self.dis_y.cleargrads()
loss_dis_y.backward()
opt_y.update()
## discriminator for B=>A
loss_dis_x_fake = losses.loss_func_comp(
self.dis_x(y_x_copy), 0.0, self.args.dis_jitter)
if self.args.conditional_discriminator:
loss_dis_x_real = losses.loss_func_comp(
self.dis_x(F.concat([x, y])), 1.0,
self.args.dis_jitter)
else:
loss_dis_x_real = losses.loss_func_comp(
self.dis_x(x), 1.0, self.args.dis_jitter)
loss_dis_x = (loss_dis_x_fake + loss_dis_x_real) * 0.5
self.dis_x.cleargrads()
loss_dis_x.backward()
opt_x.update()
if self.report_start < self._iter:
chainer.report({'loss_real': loss_dis_x_real}, self.dis_x)
chainer.report({'loss_fake': loss_dis_x_fake}, self.dis_x)
chainer.report({'loss_real': loss_dis_y_real}, self.dis_y)
chainer.report({'loss_fake': loss_dis_y_fake}, self.dis_y)
# prepare next images
if (t < self.args.n_critics - 1):
x_y_images = random.sample(self._buffer_x.images,
self.args.batch_size)
x_y_copy = Variable(self.xp.concatenate(x_y_images))
y_x_images = random.sample(self._buffer_y.images,
self.args.batch_size)
y_x_copy = Variable(self.xp.concatenate(y_x_images))
batch_x = self.get_iterator('main').next()
batch_y = self.get_iterator('train_B').next()
x = Variable(self.converter(batch_x, self.device))
y = Variable(self.converter(batch_y, self.device))
def __call__(self, x, t=None):
h = self.conv1(x)
h.name = 'conv1'
h = F.relu(h)
h.name = 'relu1'
h = F.local_response_normalization(h)
h.name = 'norm1'
h = F.max_pooling_2d(h, 3, stride=2)
h.name = 'pool1'
h = self.conv2(h)
h.name = 'conv2'
h = F.relu(h)
h.name = 'relu2'
h = F.local_response_normalization(h)
h.name = 'norm2'
h = F.max_pooling_2d(h, 3, stride=2)
h.name = 'pool2'
h = self.conv3(h)
h.name = 'conv3'
h = F.relu(h)
h.name = 'relu3'
h = self.conv4(h)
h.name = 'conv4'
h = F.relu(h)
h.name = 'relu4'
h = self.conv5(h)
h.name = 'conv5'
h = F.relu(h)
h.name = 'relu5'
h = F.max_pooling_2d(h, 3, stride=2)
h.name = 'pool5'
h = self.fc6(h)
h.name = 'fc6'
h = F.relu(h)
h.name = 'relu6'
h = F.dropout(h)
h.name = 'drop6'
h = self.fc7(h)
h.name = 'fc7'
h = F.relu(h)
h.name = 'relu7'
h = F.dropout(h)
h.name = 'drop7'
h = self.fc8(h)
h.name = 'fc8'
h = F.softmax(h)
h.name = 'prob'
self.score = h
if t is None:
assert not self.train
return h
self.loss = F.softmax_cross_entropy(self.score, t)
self.accuracy = F.accuracy(self.score, t)
chainer.report({'loss': self.loss, 'accuracy': self.accuracy})
return self.loss
def __call__(self, x, t, enable_report=True):
y = self.predictor(x)
loss = F.mean_squared_error(y, t)
if enable_report:
report({'loss': loss}, self)
return loss
def loss_gen(self, gen, y_fake):
batchsize = len(y_fake)
#fake label 1
loss = F.sum(F.softplus(-y_fake)) / batchsize
chainer.report({'loss': loss}, gen)
return loss
def __call__(self, x, t):
h = self.calc(x)
loss_c = mean_absolute_error(h, t)
loss = loss_c
chainer.report({'loss': loss, 'loss_c': loss_c}, self)
return loss
def __call__(self, x, t):
h = self.fwd(x)
loss = F.mean_squared_error(h, t)
report({'loss': loss}, self)
return loss
def loss_dis(self, dis, d_fake, d_real):
loss = -F.mean((F.log(d_real) + F.log(1 - d_fake)))
chainer.report({'loss': loss}, dis)
return loss
def loss_gen(self, gen, y_fake):
batchsize = len(y_fake)
loss = F.sum(F.softplus(-y_fake)) / batchsize
chainer.report({'loss': loss}, gen)
return loss
def loss_gen_fm(self, gen, real_fm_expected, fake_fm_expected):
fm_loss = F.mean_squared_error(real_fm_expected, fake_fm_expected)
chainer.report({'fm_loss': fm_loss}, gen) # gen/loss でアクセス可能
return fm_loss
def test_report_without_reporter(self):
observer = object()
chainer.report({'x': 1}, observer)
def test_report_without_reporter(self):
observer = object()
chainer.report({'x': 1}, observer)
def __call__(self, x, t=None):
self.clear()
x = self.xp.reshape(x, (-1, 1, 28, 28))
x = chainer.Variable(x)
t = chainer.Variable(t)
# print("init memory")
# print(self.memory_value.data)
if self.train:
self.apply_memory.memory.data = self.memory.data.copy()
self.apply_memory.memory_value.data = self.memory_value.data.copy()
self.apply_memory.memory_age.data = self.memory_age.data.copy()
query = self.encoder(x)
# using external memory from here
normed_query = F.normalize(query)
self.memory = chainer.Variable( \
self.apply_memory.memory.data)
self.memory_value = chainer.Variable( \
self.apply_memory.memory_value.data)
self.memory_age = chainer.Variable( \
self.apply_memory.memory_age.data)
all_similarities = F.matmul( \
normed_query, self.memory, transb=True) # (n_batch, memory_size)
with chainer.no_backprop_mode():
# hint_pool_idxs: (n_batch, n_best)
# infered_class: (n_batch)
hint_pool_idxs, infered_class = \
self.apply_memory(all_similarities, self.memory, self.memory_value)
teacher_idxs, neg_idxs, incorrect_memory_lookup = \
pos_neg_idxs_extractor(self.memory.data,
self.memory_value.data,
hint_pool_idxs.data,
all_similarities,
t)
n_batch = len(t.data)
teacher_vals = F.select_item(all_similarities, teacher_idxs)
neg_teacher_vals = F.select_item(all_similarities, neg_idxs)
self.loss = \
F.relu(neg_teacher_vals - teacher_vals + self.alpha) \
- self.alpha
self.loss = F.sum(self.loss)
self.loss /= n_batch
if self.train:
with chainer.no_backprop_mode():
self.memory, self.memory_value, self.memory_age = \
update_memory( \
normed_query, teacher_idxs, incorrect_memory_lookup,
t, self.memory.data, \
self.memory_value.data, self.memory_age.data)
self.memory = F.normalize(self.memory)
self.apply_memory.memory.data = self.memory.data.copy()
self.apply_memory.memory_value.data = self.memory_value.data.copy(
)
self.apply_memory.memory_age.data = self.memory_age.data.copy()
with chainer.no_backprop_mode():
self.accuracy = external_memory_accuracy(infered_class, t)
chainer.report({'loss': self.loss, 'accuracy': self.accuracy}, self)
return self.loss
def __call__(self, x, t):
loss = F.softmax_cross_entropy(self.predict(x), t)
chainer.report({'loss': loss / t.shape[0]}, self)
return loss
def loss(self, x):
"""
"""
_, loss_value, recon_loss = self(x)
report({"loss": loss_value, "recon_loss": recon_loss}, self)
return loss_value
def update_core(self):
optimizer = self.get_optimizer("main")
model_main = optimizer.target
models_others = {
k: v
for k, v in self._models.items() if v is not model_main
}
iterator = self.get_iterator("main")
batch = iterator.next()
# -- split the batch to sub-batches -- #
n = len(self._models)
in_arrays_lists = {}
for i, key in enumerate(six.iterkeys(self._models)):
in_arrays_lists[key] = self.converter(batch[i::n],
self._devices[key])
# for reducing memory
for model in six.itervalues(self._models):
model.cleargrads()
losses = []
for model_key, model in six.iteritems(self._models):
x, adj = in_arrays_lists[model_key]
with function.force_backprop_mode():
with chainer.using_device(self._devices[model_key]):
z, sum_log_det_jacs = model(x, adj)
nll = model.log_prob(z, sum_log_det_jacs)
if self.two_step:
loss = self.h_nll_weight * nll[0] + nll[1]
else:
loss = nll
#loss += F.square(F.exp(model.ln_var) + F.exp(-model.ln_var))
losses.append(loss)
for model in six.itervalues(self._models):
model.cleargrads()
for loss in losses:
loss.backward(loss_scale=self.loss_scale)
for model in six.itervalues(models_others):
model_main.addgrads(model)
total_loss = 0.0
for loss in losses:
loss_in_cpu = F.copy(loss, -1)
total_loss += loss_in_cpu
average_losses = total_loss / len(losses)
chainer.report({
"neg_log_likelihood": average_losses,
"z_var": model_main.z_var
})
optimizer.update()
for model in six.itervalues(models_others):
model.copyparams(model_main)
if self.auto_new_epoch and iterator.is_new_epoch:
optimizer.new_epoch(auto=True)
def test_report_with_unregistered_observer(self):
reporter = chainer.Reporter()
observer = object()
with reporter:
with self.assertRaises(KeyError):
chainer.report({'x': 1}, observer)
def __call__(self, x, y):
self.args.append((x, y))
chainer.report({'loss': x.sum() + y.sum()}, self)
def loss_enc(self, enc, x_real, x_fake):
loss = F.mean_squared_error(x_real, x_fake)
chainer.report({'loss': loss}, enc)
return loss
def update_core(self):
xp = self.gen_g.xp
self._iter += 1
batch = self.get_iterator('main').next()
batchsize = len(batch)
w_in = self._image_size
x = xp.zeros((batchsize, 3, w_in, w_in)).astype("f")
y = xp.zeros((batchsize, 3, w_in, w_in)).astype("f")
for i in range(batchsize):
x[i, :] = xp.asarray(batch[i][0])
y[i, :] = xp.asarray(batch[i][1])
x = Variable(x)
y = Variable(y)
x_y = self.gen_g(x)
x_y_copy = self.getAndUpdateBufferX(x_y.data)
x_y_copy = Variable(x_y_copy)
x_y_x = self.gen_f(x_y)
y_x = self.gen_f(y)
y_x_copy = self.getAndUpdateBufferY(y_x.data)
y_x_copy = Variable(y_x_copy)
y_x_y = self.gen_g(y_x)
opt_g = self.get_optimizer('gen_g')
opt_f = self.get_optimizer('gen_f')
opt_x = self.get_optimizer('dis_x')
opt_y = self.get_optimizer('dis_y')
if self._learning_rate_anneal > 0 and self._iter % self._learning_rate_anneal_interval == 0:
if opt_g.alpha > self._learning_rate_anneal:
opt_g.alpha -= self._learning_rate_anneal
if opt_f.alpha > self._learning_rate_anneal:
opt_f.alpha -= self._learning_rate_anneal
if opt_x.alpha > self._learning_rate_anneal:
opt_x.alpha -= self._learning_rate_anneal
if opt_y.alpha > self._learning_rate_anneal:
opt_y.alpha -= self._learning_rate_anneal
opt_g.zero_grads()
opt_f.zero_grads()
opt_x.zero_grads()
opt_y.zero_grads()
loss_dis_y_fake = loss_func_adv_dis_fake(self.dis_y(x_y_copy))
loss_dis_y_real = loss_func_adv_dis_real(self.dis_y(y))
loss_dis_y = loss_dis_y_fake + loss_dis_y_real
chainer.report({'loss': loss_dis_y}, self.dis_y)
loss_dis_x_fake = loss_func_adv_dis_fake(self.dis_x(y_x_copy))
loss_dis_x_real = loss_func_adv_dis_real(self.dis_x(x))
loss_dis_x = loss_dis_x_fake + loss_dis_x_real
chainer.report({'loss': loss_dis_x}, self.dis_x)
loss_dis_y.backward()
loss_dis_x.backward()
opt_y.update()
opt_x.update()
loss_gen_g_adv = loss_func_adv_gen(self.dis_y(x_y))
loss_gen_f_adv = loss_func_adv_gen(self.dis_x(y_x))
loss_cycle_x = self._lambda1 * loss_func_rec_l1(x_y_x, x)
loss_cycle_y = self._lambda1 * loss_func_rec_l1(y_x_y, y)
loss_gen = self._lambda2 * loss_gen_g_adv + self._lambda2 * loss_gen_f_adv + loss_cycle_x + loss_cycle_y
loss_gen.backward()
opt_f.update()
opt_g.update()
chainer.report({'loss_rec': loss_cycle_y}, self.gen_g)
chainer.report({'loss_rec': loss_cycle_x}, self.gen_f)
chainer.report({'loss_adv': loss_gen_g_adv}, self.gen_g)
chainer.report({'loss_adv': loss_gen_f_adv}, self.gen_f)
if self._iter % 100 == 0:
img = xp.zeros((6, 3, w_in, w_in)).astype("f")
img[0, :] = x.data
img[1, :] = x_y.data
img[2, :] = x_y_x.data
img[3, :] = y.data
img[4, :] = y_x.data
img[5, :] = y_x_y.data
img = cuda.to_cpu(img)
img = self._dataset.batch_postprocess_images(img, 2, 3)
Image.fromarray(img).save(self._eval_foler + "/iter_" +
str(self._iter) + ".jpg")
def __call__(self, x, t):
h = self.predict(x)
loss = F.softmax_cross_entropy(h, t)
chainer.report({'loss': loss, 'accuracy': F.accuracy(h, t)}, self)
return loss
def __call__(self, x, t, train=True):
y = self.predictor(x, train)
self.loss = F.mean_squared_error(y, t)
report({'loss': self.loss}, self)
return self.loss
def update_core(self):
# TODO: support n_Classfier <- いる？
# TIPS: in case of experiments, set n_critic as 5 is best result.
gen_optimizer = self.get_optimizer('gen')
critic_optimizer = self.get_optimizer('critic')
clfr_optimizer = self.get_optimizer('classfier')
xp = self.generator.xp
for i in range(self.n_critic):
# grab data
batch = self.get_iterator('main').next()
batchsize = len(batch)
batch = self.converter(batch, self.device)
real_data, real_label = batch
real_label = Variable(real_label)
real_data = Variable(real_data) / 255.
# TODO: cWGANってuniformで良いんだっけ...?
z = Variable(
xp.asarray(
self.generator.make_input_z_with_label(
batchsize, real_label.data)))
# Generator
gen_data = self.generator(z)
gen_data = gen_data.reshape(batchsize, 1, 28, 28)
# Critic(Discrimintor)
critic_real = self.critic(real_data)
critic_fake = self.critic(gen_data)
# Classifier
# classifier_real = self.classifier(real_data)
# classifier_fake = self.classifier(gen_data)
# Loss
## Critic Loss
# print(critic_fake.shape, critic_real.shape, gen_data.shape, real_data.shape)
# critic_loss = F.mean(critic_fake - critic_real)
e = xp.random.uniform(0., 1.,
(batchsize, 1, 1, 1)).astype(np.float32)
x_hat = e * real_data + (1 - e) * gen_data # recreate Variable
loss_gan = F.average(critic_fake - critic_real)
# x_hat.backward(retain_grad=True, enable_double_backprop=True)
grad, = chainer.grad([self.critic(x_hat)], [x_hat],
enable_double_backprop=True)
grad = F.sqrt(F.batch_l2_norm_squared(grad))
loss_grad = self.l * F.mean_absolute_error(grad,
xp.ones_like(grad.data))
critic_loss = loss_gan + loss_grad
self.critic.cleargrads()
critic_loss.backward()
critic_optimizer.update()
chainer.report({'critic_loss': critic_loss})
chainer.report({'loss_grad': loss_grad})
chainer.report({'loss_gan': loss_gan})
batch = self.get_iterator('main').next()
batchsize = len(batch)
batch = self.converter(batch, self.device)
real_data, real_label = batch
real_label = Variable(real_label)
real_data = Variable(real_data) / 255.
z = Variable(
xp.asarray(
self.generator.make_input_z_with_label(batchsize,
real_label.data)))
# Generator
gen_data = self.generator(z)
# Critic(Discrimintor)
critic_fake = self.critic(gen_data)
# Classifier
classifier_real = self.classifier(real_data)
classifier_fake = self.classifier(gen_data)
## Categorical Loss
c_f_loss = F.softmax_cross_entropy(classifier_fake, real_label)
c_r_loss = F.softmax_cross_entropy(classifier_real, real_label)
c_loss = (c_r_loss + c_f_loss) / 2
self.classifier.cleargrads()
c_loss.backward()
clfr_optimizer.update()
chainer.report({'c_r_loss': c_r_loss})
chainer.report({'c_loss': c_loss})
# Generator Loss
gen_loss = F.average(-critic_fake)
self.generator.cleargrads()
gen_loss.backward()
gen_optimizer.update()
chainer.report({'gen_loss': gen_loss})
self.classifier.cleargrads()
c_f_loss.backward()
gen_optimizer.update()
chainer.report({'c_f_loss': c_f_loss})
def __call__(self, x, t):
self.y = self.forward(x)
self.loss = F.softmax_cross_entropy(self.y, t)
self.accuracy = F.accuracy(self.y, t)
chainer.report({'loss': self.loss, 'acc': self.accuracy}, self)
return self.loss
def __call__(self, *args):
density = self.predictor(*args)
nll = -F.sum(F.log(density))
report({'nll': nll}, self)
return nll
def __call__(self, x, t):
y = self.predictor(x)
loss = F.softmax_cross_entropy(y, t)
accuracy = F.accuracy(y, t)
report({'loss': loss, 'accuracy': accuracy}, self)
return loss
