def decode(self, latent, sigmoid=True):
dense_0_decoded = self.decoder_dense_0(latent)
dense_1_decoded = self.decoder_dense_1(dense_0_decoded)
reshaped_decoded = F.reshape(
dense_1_decoded,
(len(dense_1_decoded), self.conv_channels[3], 8, 8))
up_4_decoded = F.upsampling_2d(reshaped_decoded,
cp.ones(reshaped_decoded.shape),
ksize=2,
cover_all=False)
deconv_3_decoded = F.relu(self.decoder_conv_3(up_4_decoded))
up_3_decoded = F.upsampling_2d(deconv_3_decoded,
cp.ones(deconv_3_decoded.shape),
ksize=2,
cover_all=False)
deconv_2_decoded = F.relu(self.decoder_conv_2(up_3_decoded))
up_2_decoded = F.upsampling_2d(deconv_2_decoded,
cp.ones(deconv_2_decoded.shape),
ksize=2,
cover_all=False)
out_img = self.decoder_conv_1(up_2_decoded)
# need the check because the bernoulli_nll has a sigmoid in it
if sigmoid:
return F.sigmoid(out_img)
else:
return out_img
def Decoder2(self,y,y1,y2,y3,y4,indexes,indexes1,indexes2,indexes3,indexes4):
y = F.upsampling_2d(y, indexes, ksize=2, stride=2, cover_all=False)
y = F.relu(self.Dbn5_3(self.Dconv5_3(y)))
y = F.relu(self.Dbn5_2(self.Dconv5_2(y)))
y = F.relu(self.Dbn5_1(self.Dconv5_1(y)))
y = F.upsampling_2d(y, indexes1, ksize=2, stride=2, cover_all=False)
Upsam2 = y
y = F.concat((y, y1), axis=1) # U-Net:Encoderの出力をDecoderの入力に連結
y = F.relu(self.Dbn4_3(self.Dconv4_3(y)))
y = F.relu(self.Dbn4_2(self.Dconv4_2(y)))
y = F.relu(self.Dbn4_1(self.Dconv4_1(y)))
feat_Dconv4_1 = y
y = F.upsampling_2d(y, indexes2, ksize=2, stride=2, cover_all=False)
Upsam3 = y
y = F.concat((y, y2), axis=1) # U-Net:Encoderの出力をDecoderの入力に連結
y = F.relu(self.Dbn3_3(self.Dconv3_3(y)))
y = F.relu(self.Dbn3_2(self.Dconv3_2(y)))
y = F.relu(self.Dbn3_1(self.Dconv3_1(y)))
feat_Dconv3_1 = y
y = F.upsampling_2d(y, indexes3, ksize=2, stride=2, cover_all=False)
Upsam4 = y
y = F.concat((y, y3), axis=1) # U-Net:Encoderの出力をDecoderの入力に連結
y = F.relu(self.Dbn2_2(self.Dconv2_2(y)))
y = F.relu(self.Dbn2_1(self.Dconv2_1(y)))
feat_Dconv2_1 = y
return y
def decode(self, z):
with chainer.using_config('train', self.train):
h = F.relu(self.dec_bn1(self.dec_conv1(z)))
h = F.upsampling_2d(h, self.p3.indexes, self.p3.kh, self.p3.sy, self.p3.ph, self.h3_size[2:])
h = F.relu(self.dec_bn2(self.dec_conv2(h)))
h = F.upsampling_2d(h, self.p2.indexes, self.p2.kh, self.p2.sy, self.p2.ph, self.h2_size[2:])
h = F.tanh(self.dec_bn3(self.dec_conv3(h)))
h = F.upsampling_2d(h, self.p1.indexes, self.p1.kh, self.p1.sy, self.p1.ph, self.h1_size[2:])
return self.dec_conv4_mu(h), self.dec_conv4_ln_var(h)
def f(x):
y = F.upsampling_2d(x,
self.indices,
ksize=self.ksize,
stride=self.stride,
outsize=self.in_shape[2:])
return y * y
def __call__(self, x, test=False):
x = F.upsampling_2d(x, ksize=(3, 1)) #129,128
print(x.data.shape)
if (self.rand):
if (U.rand_flag()):
x = U.cut_out(x)
h0 = self.conv(x)
if (self.initialBatchNorm):
h0 = self.bnorm(h0, finetune=test)
else:
h0 = F.relu(h0)
h0 = self.pool(h0, ksize=2, stride=2) #64,64
print(h0.data.shape)
h1 = self.block1(h0, test=test)
h1 = self.pool(h1, ksize=2, stride=2) #32,32
h2 = self.block2(h1, test=test)
h2 = self.pool(h2, ksize=2, stride=2) #16,16
h3 = self.block3(h2, test=test)
h3 = self.pool(h3, ksize=2, stride=2) #8,8
h4 = self.block4(h3, test=test)
h4 = self.pool(h4, ksize=2, stride=2) #4,4
h5 = self.block5(h4, test=test)
h5 = self.pool(h5, ksize=4, stride=4) #1,1
y = self.line(h5)
if not test:
self.counter += 1
return y
def decode(self, x, **kwargs):
x = self.adapt(x)
# if not kwargs.get('inference'):
# print(f'D{self.name} in:', F.min(x), F.max(x), ' '*10, end='\r')
if self.use_indices:
if not x.shape[0] == self.indexes.shape[0]:
self.indexes = self.xp.repeat(self.indexes,
x.shape[0] //
self.indexes.shape[0],
axis=0)
h = F.upsampling_2d(x, self.indexes, ksize=2, outsize=self.insize)
else:
h = F.unpooling_2d(x, ksize=2, outsize=self.insize)
y = self.dec(h)
if self.activation_d:
if self.batch_norm:
y = self.bnd(y)
y = self.activation_d(y)
if kwargs.get('show_shape'):
print(f'layer(D{self.name}): in: {x.shape} out: {y.shape}')
return y
def _upsampling_2d(self, x, indices):
if x.shape != indices.shape:
min_h = min(x.shape[2], indices.shape[2])
min_w = min(x.shape[3], indices.shape[3])
x = x[:, :, :min_h, :min_w]
indices = indices[:, :, :min_h, :min_w]
outsize = (x.shape[2] * 2, x.shape[3] * 2)
return F.upsampling_2d(x, indices, ksize=2, stride=2, outsize=outsize)
def f(x):
return F.upsampling_2d(x,
self.p.indexes,
ksize=(self.p.kh, self.p.kw),
stride=(self.p.sy, self.p.sx),
pad=(self.p.ph, self.p.pw),
outsize=self.in_shape[2:],
cover_all=self.p.cover_all)
def _upsampling_2d(self, x, pool):
if x.shape != pool.indexes.shape:
min_h = min(x.shape[2], pool.indexes.shape[2])
min_w = min(x.shape[3], pool.indexes.shape[3])
x = x[:, :, :min_h, :min_w]
pool.indexes = pool.indexes[:, :, :min_h, :min_w]
outsize = (x.shape[2] * 2, x.shape[3] * 2)
return F.upsampling_2d(
x, pool.indexes, ksize=(pool.kh, pool.kw),
stride=(pool.sy, pool.sx), pad=(pool.ph, pool.pw), outsize=outsize)
def _upsampling_2d(self, x, pool):
if x.shape != pool.indexes.shape:
min_h = min(x.shape[2], pool.indexes.shape[2])
min_w = min(x.shape[3], pool.indexes.shape[3])
x = x[:, :, :min_h, :min_w]
pool.indexes = pool.indexes[:, :, :min_h, :min_w]
outsize = (x.shape[2] * 2, x.shape[3] * 2)
return F.upsampling_2d(
x, pool.indexes, ksize=(pool.kh, pool.kw),
stride=(pool.sy, pool.sx), pad=(pool.ph, pool.pw), outsize=outsize)
def _upsampling_2d(self, x, indices):
if x.shape != indices.shape:
min_h = min(x.shape[2], indices.shape[2])
min_w = min(x.shape[3], indices.shape[3])
x = x[:, :, :min_h, :min_w]
indices = indices[:, :, :min_h, :min_w]
outsize = (x.shape[2] * 2, x.shape[3] * 2)
# appears in mixed16
if hasattr(indices, 'array'):
indices = indices.array
return F.upsampling_2d(x, indices, ksize=2, stride=2, outsize=outsize)
def check_forward(self, y):
y = F.upsampling_2d(
self.pooled_y, self.p.indexes, ksize=(self.p.kh, self.p.kw),
stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
outsize=self.in_shape[2:], cover_all=self.p.cover_all)
if isinstance(y.data, numpy.ndarray):
y = conv.im2col_cpu(y.data, self.p.kh, self.p.kw,
self.p.sy, self.p.sx, self.p.ph, self.p.pw)
else:
y = conv.im2col_gpu(y.data, self.p.kh, self.p.kw,
self.p.sy, self.p.sx, self.p.ph, self.p.pw)
for i in numpy.ndindex(y.shape):
n, c, ky, kx, oy, ox = i
up_y = y[n, c, ky, kx, oy, ox]
if ky * y.shape[3] + kx == self.p.indexes[n, c, oy, ox]:
in_y = self.pooled_y.data[n, c, oy, ox]
testing.assert_allclose(in_y, up_y)
else:
testing.assert_allclose(up_y, 0)
def check_forward(self, y):
y = F.upsampling_2d(
self.pooled_y, self.p.indexes, ksize=(self.p.kh, self.p.kw),
stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
outsize=self.in_shape[2:], cover_all=self.p.cover_all)
if isinstance(y.data, numpy.ndarray):
y = conv.im2col_cpu(y.data, self.p.kh, self.p.kw,
self.p.sy, self.p.sx, self.p.ph, self.p.pw)
else:
y = conv.im2col_gpu(y.data, self.p.kh, self.p.kw,
self.p.sy, self.p.sx, self.p.ph, self.p.pw)
for i in numpy.ndindex(y.shape):
n, c, ky, kx, oy, ox = i
up_y = y[n, c, ky, kx, oy, ox]
if ky * y.shape[3] + kx == self.p.indexes[n, c, oy, ox]:
in_y = self.pooled_y.data[n, c, oy, ox]
testing.assert_allclose(in_y, up_y)
else:
testing.assert_allclose(up_y, 0)
def check_forward(self, y):
y = F.upsampling_2d(
self.pooled_y, self.indices, ksize=self.ksize,
stride=self.stride, outsize=self.in_shape[2:])
if isinstance(y.array, numpy.ndarray):
y = conv.im2col_cpu(
y.array, self.ksize, self.ksize, self.stride, self.stride,
0, 0)
else:
y = conv.im2col_gpu(
y.array, self.ksize, self.ksize, self.stride, self.stride,
0, 0)
for i in numpy.ndindex(y.shape):
n, c, ky, kx, oy, ox = i
up_y = y[n, c, ky, kx, oy, ox]
if ky * y.shape[3] + kx == self.indices[n, c, oy, ox]:
in_y = self.pooled_y.array[n, c, oy, ox]
testing.assert_allclose(in_y, up_y)
else:
testing.assert_allclose(up_y, 0)
def check_forward(self, y):
y = F.upsampling_2d(self.pooled_y,
self.indices,
ksize=self.ksize,
stride=self.stride,
outsize=self.in_shape[2:])
if isinstance(y.array, numpy.ndarray):
y = conv.im2col_cpu(y.array, self.ksize, self.ksize, self.stride,
self.stride, 0, 0)
else:
y = conv.im2col_gpu(y.array, self.ksize, self.ksize, self.stride,
self.stride, 0, 0)
for i in numpy.ndindex(y.shape):
n, c, ky, kx, oy, ox = i
up_y = y[n, c, ky, kx, oy, ox]
if ky * y.shape[3] + kx == self.indices[n, c, oy, ox]:
in_y = self.pooled_y.array[n, c, oy, ox]
testing.assert_allclose(in_y, up_y)
else:
testing.assert_allclose(up_y, 0)
def activations(self, x, layer_idx):
"""Return filter activations projected back to the input space, i.e.
images with shape (n_feature_maps, 3, 224, 224) for a particula layer.
The layer index is expected to be 0-based.
"""
if x.shape[0] != 1:
raise TypeError(
'Visualization is only supported for a single image at a time')
self.check_add_deconv_layers()
hs, unpooling_sizes = self.feature_map_activations(x)
hs = [h.data for h in hs]
activation_maps = []
n_activation_maps = hs[layer_idx].shape[1]
xp = self.xp
for i in range(n_activation_maps): # For each channel
h = hs[layer_idx].copy()
condition = xp.zeros_like(h)
condition[0][i] = 1 # Keep one feature map and zero all other
h = Variable(xp.where(condition, h, xp.zeros_like(h)))
for i in reversed(range(layer_idx + 1)):
p = self.mps[i]
h = F.upsampling_2d(h, p.indexes, p.kh, p.sy, p.ph,
unpooling_sizes[i])
for deconv in reversed(self.deconv_blocks[i]):
h = deconv(F.relu(h))
activation_maps.append(h.data)
return xp.concatenate(activation_maps)
def activations(self, x, layer_idx):
"""Return filter activations projected back to the input space, i.e.
images with shape (n_feature_maps, 3, 224, 224) for a particula layer.
The layer index is expected to be 0-based.
"""
if x.shape[0] != 1:
raise TypeError('Visualization is only supported for a single image at a time')
self.check_add_deconv_layers()
hs, unpooling_sizes = self.feature_map_activations(x)
hs = [h.data for h in hs]
activation_maps = []
n_activation_maps = hs[layer_idx].shape[1]
xp = self.xp
for i in range(n_activation_maps): # For each channel
h = hs[layer_idx].copy()
condition = xp.zeros_like(h)
condition[0][i] = 1 # Keep one feature map and zero all other
h = Variable(xp.where(condition, h, xp.zeros_like(h)))
for i in reversed(range(layer_idx+1)):
p = self.mps[i]
h = F.upsampling_2d(h, p.indexes, p.kh, p.sy, p.ph, unpooling_sizes[i])
for deconv in reversed(self.deconv_blocks[i]):
h = deconv(F.relu(h))
activation_maps.append(h.data)
return xp.concatenate(activation_maps)
def __call__(self, x): return functions.upsampling_2d(x, self.indexes, self.ksize, self.stride, self.pad, self.outsize, self.cover_all)
def __call__(self, x): return functions.upsampling_2d(x, self.indexes, self.ksize, self.stride, self.pad, self.outsize, self.cover_all)
Created on Sat Jul 21 22:59:41 2018
@author: user
"""
import chainer
import numpy as np
import chainer.functions as F
x = np.arange(1, 37).reshape(1, 1, 6, 6).astype(np.float32)
x = chainer.Variable(x)
print(x)
pooled_x, indexes = F.max_pooling_2d(x, ksize=2, stride=2, return_indices=True)
print(pooled_x)
print(indexes)
upsampled_x = F.upsampling_2d(pooled_x,
indexes,
ksize=2,
stride=2,
outsize=x.shape[2:])
print(upsampled_x.shape)
print(upsampled_x.data)
upsampled_x = F.unpooling_2d(pooled_x, ksize=2, stride=2, outsize=x.shape[2:])
print(upsampled_x.shape)
print(upsampled_x.data)
# KerasのupsamplingはChainerのunpooling
# Chainerのupsamplingはindexesがないと動かない
def guided_backprop(model, in_array, layer, top_n=3):
xp = cuda.get_array_module(in_array)
# Forward propagation
if len(in_array.shape) == 3:
in_array = in_array.reshape(1, in_array.shape[0],
in_array.shape[1], in_array.shape[2])
acts = collections.OrderedDict()
pools = collections.OrderedDict()
h = in_array
for key, funcs in six.iteritems(model.functions):
for func in funcs:
if isinstance(func, L.BatchNormalization):
h = func(h, test=not model.train)
elif func is F.dropout:
h = func(h, ratio=model.dr, train=model.train)
elif isinstance(func, F.MaxPooling2D):
func.use_cudnn = False
pools[key] = func
h = func(h)
else:
h = func(h)
acts[key] = h.data
# Guided backpropagation w.r.t each channel
img_gbp = collections.OrderedDict()
acts_keys = list(acts.keys())
layer_ind = acts_keys.index(layer)
max_acts = collections.OrderedDict()
for ch in range(acts[layer].shape[1]):
fmap = acts[layer][0][ch]
max_acts[ch] = fmap.max()
max_chs = sorted(max_acts.items(), key=lambda x: x[1], reverse=True)
for _ch in max_chs[:top_n]:
ch = _ch[0]
fmap = acts[layer][0][ch]
max_loc = fmap.argmax()
row = int(max_loc / fmap.shape[0])
col = int(max_loc % fmap.shape[0])
one_hot = xp.zeros_like(acts[layer])
one_hot[0][ch][row][col] = 1
gx = one_hot * acts[layer]
for key in reversed(acts_keys[:layer_ind + 1]):
if reg_pool.match(key):
ckey = acts_keys[acts_keys.index(key) - 1]
gx = F.upsampling_2d(gx, pools[key].indexes, pools[key].kh,
pools[key].sy, pools[key].ph,
acts[ckey].shape[2:]).data
if reg_conv.match(key):
gx = gx * (gx > 0) * (acts[key] > 0)
gx = F.deconvolution_2d(gx, model[key].W.data,
stride=model[key].stride,
pad=model[key].pad).data
img_gbp[ch] = gx
return img_gbp, acts['prob'][0].argmax()
def dtd(model, in_array, label, eps=1e-9, lowest=0.0, highest=1.0):
xp = cuda.get_array_module(in_array)
# Forward propagation
if len(in_array.shape) == 3:
in_array = in_array.reshape(1, in_array.shape[0], in_array.shape[1],
in_array.shape[2])
acts = collections.OrderedDict()
pools = collections.OrderedDict()
h = in_array
acts['input'] = in_array
for key, funcs in six.iteritems(model.functions):
for func in funcs:
if isinstance(func, L.BatchNormalization):
h = func(h, test=not model.train)
elif func is F.dropout:
h = func(h, ratio=model.dr, train=model.train)
elif isinstance(func, F.MaxPooling2D):
func.use_cudnn = False
pools[key] = func
h = func(h)
else:
h = func(h)
acts[key] = h.data
# Layer-wise propagation by deep taylor decomposition
img_rel = xp.empty(
(acts['prob'].shape[1], in_array.shape[0], in_array.shape[1],
in_array.shape[2], in_array.shape[3]))
acts_keys = list(acts.keys())
for cls in range(acts['prob'].shape[1]):
for key in reversed(acts):
if key == 'prob':
continue
pre_key = acts_keys[acts_keys.index(key) - 1]
if key == 'out_layer':
v = xp.maximum(0, model['out_layer'].W.data)
z = xp.dot(v[cls], acts[pre_key][0])
s = acts['prob'][0][cls] / z
r = v[cls] * acts[pre_key][0] * s
if reg_fc.match(key):
if reg_pool.match(pre_key):
pre_act = acts[pre_key].reshape(acts[pre_key].size)
else:
pre_act = acts[pre_key]
v = xp.maximum(0, model[key].W.data)
z = xp.dot(v, pre_act)
s = r / (z + xp.copysign(eps, z))
c = xp.dot(s, v)
r = pre_act * c
if reg_pool.match(key):
if r.ndim == 1:
r = r.reshape(acts[key].shape)
r = F.upsampling_2d(r, pools[key].indexes, pools[key].kh,
pools[key].sy, pools[key].ph,
acts[pre_key].shape[2:]).data
if reg_conv.match(key):
v = xp.maximum(0, model[key].W.data)
if pre_key == 'input':
w = model[key].W.data
u = xp.minimum(0, model[key].W.data)
l_map = (xp.ones_like(acts['input']) * lowest).astype('f')
h_map = (xp.ones_like(acts['input']) * highest).astype('f')
z = F.convolution_2d(acts['input'], w).data -\
F.convolution_2d(l_map, v).data -\
F.convolution_2d(h_map, u).data
s = r / (z + xp.copysign(eps, z))
r = acts['input'] * F.deconvolution_2d(s, w).data -\
l_map * F.deconvolution_2d(s, v).data - \
h_map * F.deconvolution_2d(s, u).data
break
else:
z = F.convolution_2d(acts[pre_key], v).data
s = r / (z + xp.copysign(eps, z))
r = acts[pre_key] * F.deconvolution_2d(s, v).data
img_rel[cls] = r
return img_rel, acts['prob'].argmax()
def f(x):
y = F.upsampling_2d(
x, self.p.indexes, ksize=(self.p.kh, self.p.kw),
stride=(self.p.sy, self.p.sx), pad=(self.p.ph, self.p.pw),
outsize=self.in_shape[2:], cover_all=self.p.cover_all)
return y * y
def trace(self, x):
return F.upsampling_2d(
x, self.indexes, self.kh, self.sy, self.ph, self.pre_pooling_size)
def f(x):
y = F.upsampling_2d(
x, self.indices, ksize=self.ksize,
stride=self.stride, outsize=self.in_shape[2:])
return y * y
