def forward(x_data, y_data, print_conf_matrix=False):
'''
Neural net architecture
:param x_data:
:param y_data:
:param train:
:return:
'''
x, t = Variable(x_data), Variable(y_data)
h1 = F.relu(model.l1(x))
h1 = F.max_pooling_2d(h1,max_pool_window_1,stride=max_pool_stride_1)
h2 = F.dropout(F.relu(model.l2(h1)))
h2 = F.average_pooling_2d(h2, avg_pool_window_2, stride=avg_pool_stride_2)
h2 = F.max_pooling_2d(h2,max_pool_window_2,stride=max_pool_stride_2)
y = model.l3(h2)
# display confusion matrix
if print_conf_matrix:
pdb.set_trace()
print confusion_matrix(cuda.to_cpu(t.data), cuda.to_cpu(y.data).argmax(axis=1))
return F.softmax_cross_entropy(y, t), F.accuracy(y, t)
def forward(self, x_data, y_data, train=True, models=None):
VGG_mini = models["VGG_mini"]
VGG_mini2 = models["VGG_mini2"]
VGG_mini3 = models["VGG_mini3"]
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.relu(self.conv1_3(h))
h = F.relu(self.conv1_4(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(h, ratio=0.25, train=train)
h = F.relu(self.conv1_5(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(h, ratio=0.25, train=train)
h = self.fc(h)
if train:
return F.softmax_cross_entropy(h, t), F.accuracy(h, t)
else:
# return F.softmax_cross_entropy(h, t), F.accuracy(h, t), h
return F.softmax_cross_entropy(h, t), F.accuracy(h, t)
def __call__(self, x, t):
h = F.relu(self.bn1_1(self.conv1_1(x), test=not self.train))
h = F.relu(self.bn1_2(self.conv1_2(h), test=not self.train))
h = F.max_pooling_2d(h, 2, 2)
h = F.dropout(h, ratio=0.25, train=self.train)
h = F.relu(self.bn2_1(self.conv2_1(h), test=not self.train))
h = F.relu(self.bn2_2(self.conv2_2(h), test=not self.train))
h = F.max_pooling_2d(h, 2, 2)
h = F.dropout(h, ratio=0.25, train=self.train)
h = F.relu(self.bn3_1(self.conv3_1(h), test=not self.train))
h = F.relu(self.bn3_2(self.conv3_2(h), test=not self.train))
h = F.relu(self.bn3_3(self.conv3_3(h), test=not self.train))
h = F.relu(self.bn3_4(self.conv3_4(h), test=not self.train))
h = F.max_pooling_2d(h, 2, 2)
h = F.dropout(h, ratio=0.25, train=self.train)
h = F.dropout(F.relu(self.fc4(h)), ratio=0.5, train=self.train)
h = F.dropout(F.relu(self.fc5(h)), ratio=0.5, train=self.train)
h = self.fc6(h)
self.pred = F.softmax(h)
self.loss = F.softmax_cross_entropy(h, t)
self.accuracy = F.accuracy(self.pred, t)
if self.train:
return self.loss
else:
return self.pred
def __call__(self, x, test=False):
self.hiddens = []
h = self.convunit(x)
self.hiddens.append(h)
h = self.resconv0(h)
self.hiddens.append(h)
h = self.resconv1(h)
h = F.max_pooling_2d(h, (2, 2)) # 32 -> 16
self.hiddens.append(h)
h = self.resconv2(h)
self.hiddens.append(h)
h = self.resconv3(h)
h = F.max_pooling_2d(h, (2, 2)) # 16 -> 8
self.hiddens.append(h)
h = self.resconv4(h)
self.hiddens.append(h)
h = self.resconv5(h)
h = F.max_pooling_2d(h, (2, 2)) # 8 -> 4
self.hiddens.append(h)
h = self.linear0(h)
h = self.bn0(h)
self.hiddens.append(h)
y = self.linear1(h)
return y
def __call__(self,x,y,state,train=True,target=True):
h = Variable(x.reshape(len(x), 1, 4, 3), volatile=not train)
t = Variable(y.flatten(), volatile=not train)
h0 = F.max_pooling_2d(F.relu(self.conv1(h)), 2)
h0 = F.max_pooling_2d(F.relu(self.conv2(h0)), 2)
if target == False:
data = h0.data
self.data_first.append(data)
h1= F.dropout(F.relu(self.l1(h0)),ratio=0.5,train=train)
if target == False:
data = h1.data
self.data_hidden.append(data)
y = self.l2(h1)
if target ==False:
data = y.data
self.data_output.append(data)
self.loss = F.softmax_cross_entropy(y,t)
return state, self.loss
def forward(self, x_img, x_doc, y_data, train=True):
x_img = cuda.cupy.asarray(x_img)
x_doc = cuda.cupy.asarray(x_doc)
y_data = cuda.cupy.asarray(y_data)
img, doc, t = Variable(x_img), Variable(x_doc), Variable(y_data)
h = F.max_pooling_2d(F.relu(self.conv1(img)), ksize=3, stride=2, pad=0)
h = F.local_response_normalization(h)
h = F.max_pooling_2d(F.relu(self.conv2(h)), ksize=3, stride=2, pad=0)
h = F.local_response_normalization(h)
h = F.relu(self.conv3(h))
h = F.relu(self.conv4(h))
h = F.max_pooling_2d(F.relu(self.conv5(h)), ksize=3, stride=2, pad=0)
h = F.dropout(F.relu(self.fc6(h)), train=train, ratio=0.5)
h = F.dropout(F.relu(self.fc7(h)), train=train, ratio=0.5)
h2 = F.relu(self.doc_fc1(doc))
h2 = F.relu(self.doc_fc2(h2))
b = F.relu(self.bi1(h, h2))
y = self.fc8(b)
if train:
return F.softmax_cross_entropy(y, t)
else:
return F.accuracy(y, t)
def __call__(self, x, train=True):
h1 = F.relu(self.conv1_1(x))
h2 = F.max_pooling_2d(h1, 2, stride=2)
h3 = F.relu(self.conv2_1(h2))
h4 = F.max_pooling_2d(h3, 2, stride=2)
h5 = F.relu(self.conv3_1(h4))
h6 = F.relu(self.conv3_2(h5))
h7 = F.max_pooling_2d(h6, 2, stride=2)
h8 = F.relu(self.conv4_1(h7))
h9 = F.relu(self.conv4_2(h8))
h10 = F.max_pooling_2d(h9, 2, stride=2)
h11 = F.relu(self.conv5_1(h10))
h12 = F.relu(self.conv5_2(h11))
h13 = F.max_pooling_2d(h12, 2, stride=2)
h14 = F.dropout(F.relu(self.fc6(h13)), train=train, ratio=0.5)
h15 = F.dropout(F.relu(self.fc7(h14)), train=train, ratio=0.5)
h16 = self.fc8(h15)
y = F.sigmoid(h16)
return y
def forward_single(x_data, _size, train=False):
datum = x_data[0].transpose([1, 2, 0]) / 255.0
datum = datum.transpose([2, 0, 1])
c, h, w = datum.shape
datum = datum.reshape([1, c, h, w])
x = Variable(datum)
h = model.conv1(x)
h = model.norm1(h)
h = F.relu(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = model.conv2(h)
h = model.norm2(h)
h = F.relu(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = model.conv3(h)
h = model.norm3(h)
h = F.relu(h)
h = F.average_pooling_2d(h, 3, stride=2)
h = model.conv4(h)
h = F.softmax(h)
y = h.data
""" positive 領域 """
fmap = resize(y[0][1], _size).astype(np.float32)
return fmap
def compute(s):
datum = x_data[0].transpose([1, 2, 0]) / 255.0
datum = rescale(datum, s).astype(np.float32)
datum = datum.transpose([2, 0, 1])
c, h, w = datum.shape
datum = datum.reshape([1, c, h, w])
x = Variable(datum)
h = model.conv1(x)
h = model.norm1(h)
h = F.relu(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = model.conv2(h)
h = model.norm2(h)
h = F.relu(h)
h = F.max_pooling_2d(h, 3, stride=2)
h = model.conv3(h)
h = model.norm3(h)
h = F.relu(h)
h = F.average_pooling_2d(h, 3, stride=2)
h = model.conv4(h)
h = F.softmax(h)
y = h.data
""" positive 領域 """
fmap = resize(y[0][1], _size).astype(np.float32)
global_output.append(fmap)
def forward(self, x_data, y_data, train=True):
x = chainer.Variable(x_data, volatile=not train)
t = chainer.Variable(y_data, volatile=not train)
h = F.max_pooling_2d(
F.relu(self.norm1(self.conv1(x))), 3, stride=2, pad=1)
h = F.max_pooling_2d(
F.relu(self.norm2(self.conv2(h))), 3, stride=2, pad=1)
h = self.inc3a(h)
h = self.inc3b(h)
h = self.inc3c(h)
h = self.inc4a(h)
a = F.average_pooling_2d(h, 5, stride=3)
a = F.relu(self.norma(self.conva(a)))
a = F.relu(self.norma2(self.lina(a)))
a = self.outa(a)
a = F.softmax_cross_entropy(a, t)
h = self.inc4b(h)
h = self.inc4c(h)
h = self.inc4d(h)
b = F.average_pooling_2d(h, 5, stride=3)
b = F.relu(self.normb(self.convb(b)))
b = F.relu(self.normb2(self.linb(b)))
b = self.outb(b)
b = F.softmax_cross_entropy(b, t)
h = self.inc4e(h)
h = self.inc5a(h)
h = F.average_pooling_2d(self.inc5b(h), 7)
h = self.out(h)
return 0.3 * (a + b) + F.softmax_cross_entropy(h, t), F.accuracy(h, t)
def test_forward_output_size_zero(self, backend_config):
if backend_config.use_chainerx:
# TODO(sonots): Support it
if self.dtype == numpy.float16:
raise unittest.SkipTest('ChainerX does not support float16')
with self.assertRaises(Exception):
x = numpy.random.rand(4, 4, 1, 4).astype(self.dtype)
# TODO(sonots): Cleanup to use testing.backend.get_array after
# chainerx.asfortranarray is implemented.
if (backend_config.use_cuda
or (backend_config.use_chainerx
and backend_config.chainerx_device.startswith('cuda:'))):
x = cuda.to_gpu(x)
if backend_config.use_chainerx:
x = chainer.backend.to_chainerx(x)
x = chainer.Variable(x)
with backend_config:
functions.max_pooling_2d(x, 3, stride=2)
with self.assertRaises(Exception):
x = numpy.random.rand(4, 4, 4, 1).astype(self.dtype)
# TODO(sonots): Cleanup to use testing.backend.get_array after
# chainerx.asfortranarray is implemented.
if (backend_config.use_cuda
or (backend_config.use_chainerx
and backend_config.chainerx_device.startswith('cuda:'))):
x = cuda.to_gpu(x)
if backend_config.use_chainerx:
x = chainer.backend.to_chainerx(x)
x = chainer.Variable(x)
with backend_config:
functions.max_pooling_2d(x, 3, stride=2)
def __call__(self, x, test=False):
self.hiddens = []
h = self.conv00(x, test)
h = self.conv01(h, test)
h = self.conv02(h, test)
h = F.max_pooling_2d(h, (2, 2)) # 32 -> 16
h = self.bn0(h, test)
self.hiddens.append(h)
h = self.conv10(h, test)
h = self.conv11(h, test)
h = self.conv12(h, test)
h = F.max_pooling_2d(h, (2, 2)) # 16 -> 8
h = self.bn1(h, test)
self.hiddens.append(h)
h = self.conv20(h, test) # 8 -> 6
self.hiddens.append(h)
h = self.conv21(h, test)
h = self.conv22(h, test)
h = F.average_pooling_2d(h, (6, 6)) # 6 -> 1
h = self.bn2(h, test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def forward(self, x_data, y_data, train=True):
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv2_1(h))
h = F.relu(self.conv2_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv3_1(h))
h = F.relu(self.conv3_2(h))
h = F.relu(self.conv3_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv4_1(h))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv5_1(h))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(F.relu(self.fc6(h)), train=train, ratio=0.6)
h = F.dropout(F.relu(self.fc7(h)), train=train, ratio=0.6)
h = self.fc8(h)
loss = F.mean_squared_error(h, t)
return loss, h
def __call__(self, x):
batch_size = x.data.shape[0]
##### common layer
h = F.leaky_relu(self.bias1(self.bn1(self.conv1(x), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias2(self.bn2(self.conv2(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias3(self.bn3(self.conv3(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias4(self.bn4(self.conv4(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias5(self.bn5(self.conv5(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias6(self.bn6(self.conv6(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias7(self.bn7(self.conv7(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias8(self.bn8(self.conv8(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias9(self.bn9(self.conv9(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias10(self.bn10(self.conv10(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias11(self.bn11(self.conv11(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias12(self.bn12(self.conv12(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias13(self.bn13(self.conv13(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.max_pooling_2d(h, ksize=2, stride=2, pad=0)
h = F.leaky_relu(self.bias14(self.bn14(self.conv14(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias15(self.bn15(self.conv15(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias16(self.bn16(self.conv16(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias17(self.bn17(self.conv17(h), test=not self.train, finetune=self.finetune)), slope=0.1)
h = F.leaky_relu(self.bias18(self.bn18(self.conv18(h), test=not self.train, finetune=self.finetune)), slope=0.1)
###### new layer
h = self.conv19(h)
h = F.average_pooling_2d(h, h.data.shape[-1], stride=1, pad=0)
# reshape
y = F.reshape(h, (batch_size, -1))
return y
def __call__(self, x, t):
h = F.relu(self.bn1_1(self.conv1_1(x), test=not self.train))
h = F.relu(self.bn1_2(self.conv1_2(h), test=not self.train))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.bn2_1(self.conv2_1(h), test=not self.train))
h = F.relu(self.bn2_2(self.conv2_2(h), test=not self.train))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.bn3_1(self.conv3_1(h), test=not self.train))
h = F.relu(self.bn3_2(self.conv3_2(h), test=not self.train))
h = F.relu(self.bn3_3(self.conv3_3(h), test=not self.train))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.bn4_1(self.conv4_1(h), test=not self.train))
h = F.relu(self.bn4_2(self.conv4_2(h), test=not self.train))
h = F.relu(self.bn4_3(self.conv4_3(h), test=not self.train))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.bn5_1(self.conv5_1(h), test=not self.train))
h = F.relu(self.bn5_2(self.conv5_2(h), test=not self.train))
h = F.relu(self.bn5_3(self.conv5_3(h), test=not self.train))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(F.relu(self.fc6(h)), train=self.train, ratio=0.6)
h = F.dropout(F.relu(self.fc7(h)), train=self.train, ratio=0.6)
self.pred = self.fc8(h)
if t is not None:
self.loss = F.mean_squared_error(self.pred, t)
return self.loss
else:
return self.pred
def __call__(self, x, test=False):
# add gaussian noise
#xp = cuda.get_array_module(x.data)
#with cuda.get_device_from_id(self.device):
# noise = xp.random.randn(*x.shape) * 0.0015
# x.data += noise
h = self.conv00(x, test)
h = self.conv01(h, test)
h = self.conv02(h, test)
h = F.max_pooling_2d(h, (2, 2)) # 32 -> 16
h = self.bn0(h, test)
h = F.dropout(h, train=not test)
h = self.conv10(h, test)
h = self.conv11(h, test)
h = self.conv12(h, test)
h = F.max_pooling_2d(h, (2, 2)) # 16 -> 8
h = self.bn1(h, test)
h = F.dropout(h, train=not test)
h = self.conv20(h, test) # 8 -> 6
h = self.conv21(h, test)
h = self.conv22(h, test)
h = self.conv23(h, test)
h = F.average_pooling_2d(h, (6, 6)) # 6 -> 1
h = self.bn2(h, test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def __call__(self, x):
ys = []
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv2_1(h))
h = F.relu(self.conv2_2(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv3_1(h))
h = F.relu(self.conv3_2(h))
h = F.relu(self.conv3_3(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv4_1(h))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
ys.append(self.norm4(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv5_1(h))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = F.max_pooling_2d(h, 3, stride=1, pad=1)
h = F.relu(self.conv6(h))
h = F.relu(self.conv7(h))
ys.append(h)
return ys
def forward(self, x_data, y_data, train=True, gpu=-1): x, t = Variable(x_data), Variable(y_data) h = F.max_pooling_2d(F.relu(self.conv1(x)), ksize=2, stride=2) h = F.max_pooling_2d(F.relu(self.conv2(h)), ksize=3, stride=3) h = F.dropout(F.relu(self.l3(h)), train=train) y = self.l4(h) return F.softmax_cross_entropy(y, t), F.accuracy(y,t)
def __call__(self, x):
h = x
h = self.__dict__["P1_1"](F.leaky_relu(h))
h = self.__dict__["BN1_1"](h)
h = self.__dict__["P1_2"](F.leaky_relu(h))
h = self.__dict__["BN1_2"](h)
h = F.max_pooling_2d(F.leaky_relu(h), ksize=3, stride=2, cover_all=False)
h = self.__dict__["P2_1"](h)
h = self.__dict__["BN2_1"](h)
h = self.__dict__["P2_2"](F.leaky_relu(h))
h = self.__dict__["BN2_2"](h)
h = self.__dict__["P2_2"](F.leaky_relu(h))
h = self.__dict__["BN2_3"](h)
h = F.max_pooling_2d(F.leaky_relu(h), ksize=3, stride=2, cover_all=False)
h = self.__dict__["P3_1"](h)
h = self.__dict__["BN3_1"](h)
h = self.__dict__["P3_2"](F.leaky_relu(h))
h = self.__dict__["BN3_2"](h)
h = self.__dict__["P3_3"](F.leaky_relu(h))
h = self.__dict__["BN3_3"](h)
#h = F.average_pooling_2d(F.leaky_relu(h), ksize=6)
h = F.spatial_pyramid_pooling_2d(F.leaky_relu(h), 3, F.MaxPooling2D)
h = self.__dict__["BNL0"](h)
h = self.__dict__["L1"](F.leaky_relu(h))
h = self.__dict__["BNL1"](h)
h = self.__dict__["L2"](F.leaky_relu(h))
y = h
#h = F.spatial_pyramid_pooling_2d(F.leaky_relu(h), 3)
#y = F.reshape(h,(len(h.data),self.F_unit))
return y
def __call__(self, x, train=True):
self.train = train
h = F.max_pooling_2d( F.relu(self.bn1(self.conv1(x), test=not self.train)), 3, stride=2 )
h = F.max_pooling_2d( F.relu(self.bn2(self.conv2(h), test=not self.train)), 3, stride=2 )
h = F.max_pooling_2d( F.relu(self.bn3(self.conv3(h), test=not self.train)), 3, stride=2 )
h = self.fc1(h)
return h
def __call__(self, x, test=False):
self.hiddens = []
h = self.convunit(x, test)
h = F.dropout(h, train=not test)
h = self.resconvunit0(h, test)
self.hiddens.append(h)
h = F.dropout(h, train=not test)
h = self.resconvunit1(h, test)
self.hiddens.append(h)
h = F.max_pooling_2d(h, (2, 2))
h = F.dropout(h, train=not test)
h = self.resconvunit2(h, test)
self.hiddens.append(h)
h = F.dropout(h, train=not test)
h = self.resconvunit3(h, test)
self.hiddens.append(h)
h = F.max_pooling_2d(h, (2, 2))
h = F.dropout(h, train=not test)
h = self.resconvunit4(h, test)
self.hiddens.append(h)
h = F.dropout(h, train=not test)
h = self.resconvunit5(h, test)
self.hiddens.append(h)
h = F.max_pooling_2d(h, (2, 2))
h = F.dropout(h, train=not test)
h = self.linear(h)
return h
def __call__(self, x, test=False):
# add gaussian noise
xp = cuda.get_array_module(x.data)
with cuda.get_device(self.device):
noise = xp.random.randn(*x.shape) * 0.15
x.data += noise
# (conv -> act -> bn) x 3 -> maxpool -> dropout
h = self.bn_conv0(self.act(self.conv0(x), 0.1), test)
h = self.bn_conv1(self.act(self.conv1(h), 0.1), test)
h = self.bn_conv2(self.act(self.conv2(h), 0.1), test)
h = F.max_pooling_2d(h, (2, 2)) # 32 -> 16
h = F.dropout(h, 0.5, not test)
# (conv -> act -> bn) x 3 -> maxpool -> dropout
h = self.bn_conv3(self.act(self.conv3(h), 0.1), test)
h = self.bn_conv4(self.act(self.conv4(h), 0.1), test)
h = self.bn_conv5(self.act(self.conv5(h), 0.1), test)
h = F.max_pooling_2d(h, (2, 2)) # 16 -> 8
h = F.dropout(h, 0.5, not test)
# conv -> act -> bn -> (nin -> act -> bn) x 2
h = self.bn_conv6(self.act(self.conv6(h), 0.1), test) # 8 -> 6
h = self.bn_conv7(self.act(self.conv7(h), 0.1), test)
h = self.bn_conv8(self.act(self.conv8(h), 0.1), test)
h = F.average_pooling_2d(h, (6, 6))
h = self.linear(h)
return h
def __call__(self, x, model_params, test=False):
# Initial convolution
mp_filtered = self._filter_model_params(model_params, "/convunit")
h = self.convunit(x, mp_filtered, test)
# Residual convolution
mp_filtered = self._filter_model_params(model_params, "/resconvunit0")
h = self.resconvunit0(h, mp_filtered, test)
mp_filtered = self._filter_model_params(model_params, "/resconvunit1")
h = self.resconvunit1(h, mp_filtered, test)
h = F.max_pooling_2d(h, (2, 2)) # 32 -> 16
h = F.dropout(h, train=not test)
mp_filtered = self._filter_model_params(model_params, "/resconvunit2")
h = self.resconvunit2(h, mp_filtered, test)
mp_filtered = self._filter_model_params(model_params, "/resconvunit3")
h = self.resconvunit3(h, mp_filtered, test)
h = F.max_pooling_2d(h, (2, 2)) # 16 -> 8
h = F.dropout(h, train=not test)
mp_filtered = self._filter_model_params(model_params, "/resconvunit4")
h = self.resconvunit4(h, mp_filtered, test)
mp_filtered = self._filter_model_params(model_params, "/resconvunit5")
h = self.resconvunit5(h, mp_filtered, test)
# Linear
mp_filtered = self._filter_model_params(model_params, "/linear")
y = self.linear(h, mp_filtered["/W"], mp_filtered["/b"])
return y
def extract_features(self, x, train=True):
h = F.relu(self.conv1(x))
h = F.relu(self.conv2(h))
self.h1 = h
h = F.max_pooling_2d(h,2)
h = F.relu(self.conv3(h))
h = F.relu(self.conv4(h))
self.h2 = h
h = F.max_pooling_2d(h,2)
h = F.relu(self.conv5(h))
h = F.relu(self.conv6(h))
self.h3 = h
h = F.max_pooling_2d(h,2)
h = F.relu(self.conv7(h))
h = F.relu(self.conv8(h))
self.h4 = h
h = F.max_pooling_2d(h,2)
h = F.relu(self.conv9(h))
h = F.relu(self.conv10(h))
self.h5 = h
h = F.max_pooling_2d(h,2)
return h
def __call__(self, x, t=None):
h = F.relu(self.bn1_1(self.conv1_1(x), test=not self.train))
h = F.dropout(h, ratio=0.3, train=self.train)
h = F.relu(self.bn1_2(self.conv1_2(h), test=not self.train))
h = F.max_pooling_2d(h, 3, stride=3)
h = F.relu(self.bn2_1(self.conv2_1(h), test=not self.train))
h = F.dropout(h, ratio=0.4, train=self.train)
h = F.relu(self.bn2_2(self.conv2_2(h), test=not self.train))
h = F.max_pooling_2d(h, 3, stride=3)
h = F.relu(self.bn3_1(self.conv3_1(h), test=not self.train))
h = F.dropout(h, ratio=0.4, train=self.train)
h = F.relu(self.bn3_2(self.conv3_2(h), test=not self.train))
h = F.dropout(h, ratio=0.4, train=self.train)
h = F.max_pooling_2d(h, 3, stride=3)
h = F.dropout(h, ratio=0.5, train=self.train)
h = F.relu(self.bn4(self.fc4(h), test=not self.train))
h = F.dropout(h, ratio=0.5, train=self.train)
h = F.relu(self.bn5(self.fc5(h), test=not self.train))
h = F.dropout(h, ratio=0.5, train=self.train)
h = self.fc6(h)
self.y = h
if t is not None:
self.loss = F.softmax_cross_entropy(h, t)
self.accuracy = F.accuracy(h, t)
return self.loss
self.pred = F.softmax(self.y)
return self.pred
def forward(self, x_data, y_data, train=True):
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
h = F.relu(self.bn1_1(self.conv1_1(x)))
h = F.relu(self.bn1_2(self.conv1_2(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(h, ratio=0.25, train=train)
h = F.relu(self.bn2_1(self.conv2_1(h)))
h = F.relu(self.bn2_2(self.conv2_2(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(h, ratio=0.25, train=train)
h = F.relu(self.bn3_1(self.conv3_1(h)))
h = F.relu(self.bn3_2(self.conv3_2(h)))
h = F.relu(self.bn3_3(self.conv3_3(h)))
h = F.relu(self.bn3_4(self.conv3_4(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.dropout(h, ratio=0.25, train=train)
h = F.dropout(F.relu(self.fc4(h)), train=train, ratio=0.5)
h = F.dropout(F.relu(self.fc5(h)), train=train, ratio=0.5)
h = self.fc6(h)
if train:
return F.softmax_cross_entropy(h, t), F.accuracy(h, t)
else:
return F.softmax_cross_entropy(h, t), F.accuracy(h, t), h
def __call__(self, x, rois, train=False):
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv2_1(h))
h = F.relu(self.conv2_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv3_1(h))
h = F.relu(self.conv3_2(h))
h = F.relu(self.conv3_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv4_1(h))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv5_1(h))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = roi_pooling_2d(h, rois, 7, 7, 0.0625)
h = F.dropout(F.relu(self.fc6(h)), train=train, ratio=0.5)
h = F.dropout(F.relu(self.fc7(h)), train=train, ratio=0.5)
cls_score = F.softmax(self.cls_score(h))
bbox_pred = self.bbox_pred(h)
return cls_score, bbox_pred
def forward(self, x_data, y_data, train=True, gpu=-1):
if gpu >= 0:
x_data = cuda.to_gpu(x_data)
y_data = cuda.to_gpu(y_data)
x, t = Variable(x_data), Variable(y_data)
h = F.max_pooling_2d(F.relu(self.conv1(x)), ksize=2, stride=2) # padding=0
h = F.max_pooling_2d(F.relu(self.conv2(h)), ksize=3, stride=3)
# h = F.spatial_pyramid_pooling_2d(F.relu(self.conv2(h)), 3, F.MaxPooling2D)
h = F.dropout(F.relu(self.l3(h)), train=train)
y = self.l4(h)
if train == False: # 評価時にのみ以下を実行
cnt = 0
missid = []
for ydata in y.data:
# ファイル出力して確認するなら
# fp_.write(str(np.argmax(ydata)))
# fp_.write(' ')
if y_data[cnt] != np.argmax(ydata):
# 識別に失敗したデータを出力する処理．
missid.append(glob_z_test[z_batch[cnt]])
cnt += 1
glob_all_missid.extend(missid)
# 全バッチにおいて識別失敗した id を格納
return F.softmax_cross_entropy(y, t), F.accuracy(y,t)
def forward(self, x_data, y_data, train=True):
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
h = self.prelu1_1(self.bn1_1(self.conv1_1(x)))
h = self.prelu1_2(self.bn1_2(self.conv1_2(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = self.prelu2_1(self.bn2_1(self.conv2_1(h)))
h = self.prelu2_2(self.bn2_2(self.conv2_2(h)))
h = F.max_pooling_2d(h, 2, stride=2)
h = self.prelu3_1(self.conv3_1(h))
h = self.prelu3_2(self.conv3_2(h))
h = self.prelu3_3(self.conv3_3(h))
h = F.max_pooling_2d(h, 2, stride=1)
h = self.prelu4_1(self.conv4_1(h))
h = self.prelu4_2(self.conv4_2(h))
h = self.prelu4_3(self.conv4_3(h))
h = F.max_pooling_2d(h, 2, stride=1)
h = self.prelu5_1(self.conv5_1(h))
h = self.prelu5_2(self.conv5_2(h))
h = self.prelu5_3(self.conv5_3(h))
h = F.max_pooling_2d(h, 2, stride=1)
h = F.dropout(self.prelu6(self.fc6(h)), train=train, ratio=0.5)
h = F.dropout(self.prelu7(self.fc7(h)), train=train, ratio=0.5)
h = self.fc8(h)
if train:
return F.softmax_cross_entropy(h, t), F.accuracy(h, t)
else:
return F.softmax_cross_entropy(h, t), F.accuracy(h, t), h
def forward(x, is_train=True):
h1 = F.max_pooling_2d(F.relu(model.conv1(x)), 3)
h2 = F.max_pooling_2d(F.relu(model.conv2(h1)), 3)
h3 = F.dropout(F.relu(model.l1(h2)), train=is_train)
h4 = F.dropout(F.relu(model.l2(h3)), train=is_train)
p = model.l3(h4)
return p
def extract_pool21(self, x_dep):
x2 = Variable(x_dep.reshape(1, 1, x_dep.shape[0], x_dep.shape[1]),
volatile=True)
h2 = F.max_pooling_2d(F.relu(self.bn21(self.conv21(x2))), 2, stride=2)
return h2
def _max_pooling_2d(x):
return max_pooling_2d(x, ksize=2)
def __call__(self, x):
h = F.max_pooling_2d(F.relu(self.conv1(x)), 2)
h = F.max_pooling_2d(F.relu(self.conv2(h)), 2)
return self.l1(h)
def max_pooling3(x):
return F.max_pooling_2d(x, ksize=p3_s, stride=p3_s)
def max_pooling1(x):
return F.max_pooling_2d(x, ksize=p1_s, stride=p1_s)
def __init__(self,
n_layer,
n_class=None,
initialW=None,
fc_kwargs={},
first_bn_mixed16=False):
""" CTOR. """
super(ResNet, self).__init__()
conv1_no_bias = n_layer != 50
blocks = self._blocks[n_layer]
if initialW is None:
initialW = initializers.HeNormal(scale=1., fan_option='fan_out')
if 'initialW' not in fc_kwargs:
fc_kwargs['initialW'] = initializers.Normal(scale=0.01)
kwargs = {'initialW': initialW}
with self.init_scope():
bn_kwargs = {} if not first_bn_mixed16 else {
'dtype': chainer.mixed16
}
self.conv1 = Conv2DBNActiv(3,
64,
ksize=7,
stride=2,
pad=3,
nobias=conv1_no_bias,
initialW=initialW,
bn_kwargs=bn_kwargs)
self.pool1 = lambda x: F.max_pooling_2d(x, ksize=3, stride=2)
# TODO: refactorize this part
if n_layer < 50:
last_n_channel = 512
self.res2 = ResBasicBlock(blocks[0],
64,
64,
stride=1,
**kwargs)
self.res3 = ResBasicBlock(blocks[1],
64,
128,
stride=2,
**kwargs)
self.res4 = ResBasicBlock(blocks[2],
128,
256,
stride=2,
**kwargs)
self.res5 = ResBasicBlock(blocks[3],
256,
last_n_channel,
stride=2,
**kwargs)
else:
last_n_channel = 2048
self.res2 = ResBottleneckBlock(blocks[0],
64,
64,
256,
stride=1,
**kwargs)
self.res3 = ResBottleneckBlock(blocks[1],
256,
128,
512,
stride=2,
**kwargs)
self.res4 = ResBottleneckBlock(blocks[2],
512,
256,
1024,
stride=2,
**kwargs)
self.res5 = ResBottleneckBlock(blocks[3],
1024,
512,
last_n_channel,
stride=2,
**kwargs)
self.pool5 = lambda x: F.average(x, axis=(2, 3))
self.fc6 = L.Linear(last_n_channel,
n_class,
nobias=False,
**fc_kwargs)
def _max_pooling_2d(x): return F.max_pooling_2d(x, ksize=2, stride=2, pad=0)
def encoder(self, x, batchsize, train=True):
with chainer.using_config('train', train):
x2 = F.reshape(x, (batchsize,84,84,3))
x3 = F.transpose(x2, [0,3,1,2])
c1_r=self.chain.l_conv1_r(x3)
n1_r=self.chain.l_norm1_r(c1_r)
c1_1=self.chain.l_conv1_1(x3)
n1_1=self.chain.l_norm1_1(c1_1)
a1_1=F.relu(n1_1)
c1_2=self.chain.l_conv1_2(a1_1)
n1_2=self.chain.l_norm1_2(c1_2)
a1_2=F.relu(n1_2)
c1_3=self.chain.l_conv1_3(a1_2)
n1_3=self.chain.l_norm1_3(c1_3)
a1_3=F.relu(n1_3+n1_r)
p1=F.max_pooling_2d(a1_3,2)
p1=F.dropout(p1,ratio=0.3)
c2_r=self.chain.l_conv2_r(p1)
n2_r=self.chain.l_norm2_r(c2_r)
c2_1=self.chain.l_conv2_1(p1)
n2_1=self.chain.l_norm2_1(c2_1)
a2_1=F.relu(n2_1)
c2_2=self.chain.l_conv2_2(a2_1)
n2_2=self.chain.l_norm2_2(c2_2)
a2_2=F.relu(n2_2)
c2_3=self.chain.l_conv2_3(a2_2)
n2_3=self.chain.l_norm2_3(c2_3)
a2_3=F.relu(n2_3+n2_r)
p2=F.max_pooling_2d(a2_3,2)
p2=F.dropout(p2, ratio=0.2)
c3_r=self.chain.l_conv3_r(p2)
n3_r=self.chain.l_norm3_r(c3_r)
c3_1=self.chain.l_conv3_1(p2)
n3_1=self.chain.l_norm3_1(c3_1)
a3_1=F.relu(n3_1)
c3_2=self.chain.l_conv3_2(a3_1)
n3_2=self.chain.l_norm3_2(c3_2)
a3_2=F.relu(n3_2)
c3_3=self.chain.l_conv3_3(a3_2)
n3_3=self.chain.l_norm3_3(c3_3)
a3_3=F.relu(n3_3+n3_r)
p3=F.max_pooling_2d(a3_3,2)
p3=F.dropout(p3,ratio=0.2)
c4_r=self.chain.l_conv4_r(p3)
n4_r=self.chain.l_norm4_r(c4_r)
c4_1=self.chain.l_conv4_1(p3)
n4_1=self.chain.l_norm4_1(c4_1)
a4_1=F.relu(n4_1)
c4_2=self.chain.l_conv4_2(a4_1)
n4_2=self.chain.l_norm4_2(c4_2)
a4_2=F.relu(n4_2)
c4_3=self.chain.l_conv4_3(a4_2)
n4_3=self.chain.l_norm4_3(c4_3)
a4_3=F.relu(n4_3+n4_r)
p4=F.max_pooling_2d(a4_3,2)
p4=F.dropout(p4, ratio=0.2)
p5=F.average_pooling_2d(p4,6)
h_t=F.reshape(p5, (batchsize,-1))
return h_t
def __call__(self, x):
h1 = F.max_pooling_2d(F.relu(self.conv1(x)), 2,
2) # 最大値プーリングは2×2，活性化関数はReLU
h2 = F.max_pooling_2d(F.relu(self.conv2(h1)), 2, 2)
y = self.l3(h2)
return y
def __call__(self, x):
h = F.relu(self.bn1(self.conv1(x)))
h = F.max_pooling_2d(F.relu(h), 3, stride=2, pad=0)
return h
plt.tick_params(labelbottom="off")
plt.tick_params(labelleft="off")
if ans:
plt.figure(figsize=(15,15))
serializers.load_npz('bestepoch.model', model)
if args.gpu >= 0:
model.to_gpu()
cnt = 0
for idx in range(N_test):
x_batch = x_test[idx].reshape(1, 1, imsize, imsize)
if args.gpu >= 0:
x_data = cuda.to_gpu(x_batch)
x = Variable(x_data)
h1 = F.max_pooling_2d( F.relu( model['layer{}'.format(1)](model['layer{}'.format(0)](x)) ), ksize=3, stride=3 )
h2 = F.max_pooling_2d( F.relu( model['layer{}'.format(3)](model['layer{}'.format(2)](h1)) ), ksize=3, stride=3 )
h3 = F.dropout( F.relu(model['layer{}'.format(4)](h2)), ratio=0.1, train=False)
y = model['layer{}'.format(5)](h3)
y_batch = cuda.to_cpu(y.data)
if y_test[idx] != np.argmax(y_batch):
cnt+=1
draw_digit2(x_test[idx], cnt, y_test[idx], np.argmax(y_batch))
figname = 'false_figures.png'
plt.savefig(figname)
# グラフの描画
if draw:
def __call__(self, x):
x = F.relu(self.bn1(self.conv1(x)))
x = F.max_pooling_2d(x, 2, 2)
x = F.relu(self.bn2(self.conv2(x)))
x = F.max_pooling_2d(x, 2, 2)
return self.fc(x)
def max_pooling2(x):
return F.max_pooling_2d(x, ksize=p2_s, stride=p2_s)
def predict_proba(self, x):
output_shape = (x.data.shape[0], self.n_class, x.data.shape[2],
x.data.shape[3])
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv2_1(h))
h = F.relu(self.conv2_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv3_1(h))
h = F.relu(self.conv3_2(h))
h = F.relu(self.conv3_3(h))
p3 = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv4_1(p3))
h = F.relu(self.conv4_2(h))
h = F.relu(self.conv4_3(h))
p4 = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv5_1(p4))
h = F.relu(self.conv5_2(h))
h = F.relu(self.conv5_3(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = feature_map_dropout(F.relu(self.fc6_conv(h)),
train=self.train,
ratio=0.5)
h = feature_map_dropout(F.relu(self.fc7_conv(h)),
train=self.train,
ratio=0.5)
h = F.relu(self.upconv1(h))
p4 = self.upsample_pool4(p4)
g = feature_map_dropout(self.crop(
p4, h.data.shape, self.calc_offset(p4.data.shape, h.data.shape)),
train=self.train,
ratio=0.5)
del p4
h = F.relu(self.upconv2(h + g))
p3 = self.upsample_pool3(p3)
g = feature_map_dropout(self.crop(
p3, h.data.shape, self.calc_offset(p3.data.shape, h.data.shape)),
train=self.train,
ratio=0.5)
del p3
j = F.relu(self.upconv3(h + g))
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
h = F.max_pooling_2d(h, 2, stride=2)
h = F.relu(self.conv2_1(h))
h = F.relu(self.conv2_2(h))
p2 = F.max_pooling_2d(h, 2, stride=2)
p2 = self.upsample_pool2(p2)
g = feature_map_dropout(self.crop(
p2, j.data.shape, self.calc_offset(p2.data.shape, j.data.shape)),
train=self.train,
ratio=0.5)
del p2
j = F.relu(self.upconv4(j + g))
h = F.relu(self.conv1_1(x))
h = F.relu(self.conv1_2(h))
p1 = F.max_pooling_2d(h, 2, stride=2)
p1 = self.upsample_pool1(p1)
g = feature_map_dropout(self.crop(
p1, j.data.shape, self.calc_offset(p1.data.shape, j.data.shape)),
train=self.train,
ratio=0.5)
del p1
h = F.relu(self.upconv5(j + g))
h = self.crop(h, output_shape,
self.calc_offset(h.data.shape, output_shape))
return h
def test_forward_cpu_wide(self): # see #120
x_data = numpy.random.rand(2, 3, 15, 15).astype(self.dtype)
x = chainer.Variable(x_data)
functions.max_pooling_2d(x, 6, stride=6, pad=0)
def forward(self):
x = chainer.Variable(self.x)
return functions.max_pooling_2d(x, 3, stride=2, pad=1, cover_all=False)
def _max_pooling_2d(self, x):
return F.max_pooling_2d(x, 3, stride=2)
# optimizer
optimizer = optimizers.Adam()
optimizer.setup(model)
for epoch in range(epochs):
print('start_epoch:' + str(epoch))
cycle, output = make_cycle(num_of_data)
cycle = cuda.to_gpu(cycle)
output = cuda.to_gpu(output)
for k in range(num_of_data / 2):
optimizer.zero_grads()
x, t = Variable(cycle[2 * k:2 * k + 1, :, :, :]), Variable(
output[2 * k:2 * k + 1])
x = F.relu(model.conv1(x))
x = F.max_pooling_2d(F.relu(model.conv2(x)), 2)
x = F.relu(model.conv3(x))
x = F.max_pooling_2d(F.relu(model.conv4(x)), 2)
x = F.relu(model.conv5(x))
x = F.max_pooling_2d(F.relu(model.conv6(x)), 2)
x = F.relu(model.conv7(x))
x = F.dropout(F.relu(model.l1(x)), ratio=dropout_ratio, train=True)
y = model.l2(x)
loss = F.softmax_cross_entropy(y, t)
acc = F.accuracy(y, t)
loss.backward()
optimizer.update()
cycle, output = make_batch(batch_size)
x, t = Variable(cuda.to_gpu(cycle)), Variable(cuda.to_gpu(output))
x = F.relu(model.conv1(x))
x = F.max_pooling_2d(F.relu(model.conv2(x)), 2)
def f(x):
return functions.max_pooling_2d(x,
3,
stride=2,
pad=1,
cover_all=self.cover_all)
def __call__(self, x, get_feature=False):
"""Input dims are (batch_size, 3, 299, 299)."""
# assert x.shape[1:] == (3, 299, 299)
x -= 128.0
x *= 0.0078125
h = F.relu(self.bn_conv(self.conv(x)))
# assert h.shape[1:] == (32, 149, 149)
h = F.relu(self.bn_conv_1(self.conv_1(h)))
# assert h.shape[1:] == (32, 147, 147)
h = F.relu(self.bn_conv_2(self.conv_2(h)))
# assert h.shape[1:] == (64, 147, 147)
h = F.max_pooling_2d(h, 3, stride=2, pad=0)
# assert h.shape[1:] == (64, 73, 73)
h = F.relu(self.bn_conv_3(self.conv_3(h)))
# assert h.shape[1:] == (80, 73, 73)
h = F.relu(self.bn_conv_4(self.conv_4(h)))
# assert h.shape[1:] == (192, 71, 71)
h = F.max_pooling_2d(h, 3, stride=2, pad=0)
# assert h.shape[1:] == (192, 35, 35)
h = self.mixed(h)
# assert h.shape[1:] == (256, 35, 35)
h = self.mixed_1(h)
# assert h.shape[1:] == (288, 35, 35)
h = self.mixed_2(h)
# assert h.shape[1:] == (288, 35, 35)
h = self.mixed_3(h)
# assert h.shape[1:] == (768, 17, 17)
h = self.mixed_4(h)
# assert h.shape[1:] == (768, 17, 17)
h = self.mixed_5(h)
# assert h.shape[1:] == (768, 17, 17)
h = self.mixed_6(h)
# assert h.shape[1:] == (768, 17, 17)
h = self.mixed_7(h)
# assert h.shape[1:] == (768, 17, 17)
h = self.mixed_8(h)
# assert h.shape[1:] == (1280, 8, 8)
h = self.mixed_9(h)
# assert h.shape[1:] == (2048, 8, 8)
h = self.mixed_10(h)
# assert h.shape[1:] == (2048, 8, 8)
h = F.average_pooling_2d(h, 8, 1)
# assert h.shape[1:] == (2048, 1, 1)
h = F.reshape(h, (-1, 2048))
if get_feature:
return h
else:
h = self.logit(h)
h = F.softmax(h)
# assert h.shape[1:] == (1008,)
return h
def __call__(self, x):
h1 = F.max_pooling_2d(F.relu(self.conv1(x)), ksize=2)
h2 = F.max_pooling_2d(F.relu(self.conv2(h1)), ksize=2)
h3 = F.relu(self.conv3(h2))
h4 = F.relu(self.link(h3))
return self.link_class(h4)
def recognition_net(self, images, localizations):
points = F.spatial_transformer_grid(localizations, self.target_shape)
rois = F.spatial_transformer_sampler(images, points)
connected_rois = self.data_bn(rois)
h = F.relu(self.bn0(self.conv0(connected_rois)))
h = F.max_pooling_2d(h, 3, stride=2, pad=1)
h = self.rs1_1(h)
h = self.rs1_2(h)
h = self.rs2_1(h)
h = self.rs2_2(h)
h = self.rs3_1(h)
h = self.rs3_2(h)
# h = self.rs4_1(h)
# h = self.rs4_2(h)
self.recognition_vis_anchor = h
h = F.average_pooling_2d(h, 5, stride=1)
if self.uses_original_data:
# merge data of all 4 individual images in channel dimension
batch_size, num_channels, height, width = h.shape
h = F.reshape(h,
(batch_size // 4, 4 * num_channels, height, width))
h = F.relu(self.fc1(h))
# for each timestep of the localization net do the 'classification'
h = F.reshape(h, (self.num_timesteps, -1, self.fc1.out_size))
overall_predictions = []
for timestep in F.separate(h, axis=0):
# go 2x num_labels plus 1 timesteps because of ctc loss
lstm_predictions = []
self.recognition_lstm.reset_state()
if self.use_blstm:
self.recognition_blstm.reset_state()
for _ in range(self.num_labels):
lstm_prediction = self.recognition_lstm(timestep)
lstm_predictions.append(lstm_prediction)
if self.use_blstm:
blstm_predictions = []
for lstm_prediction in reversed(lstm_predictions):
blstm_prediction = self.recognition_blstm(lstm_prediction)
blstm_predictions.append(blstm_prediction)
lstm_predictions = reversed(blstm_predictions)
final_lstm_predictions = []
for lstm_prediction in lstm_predictions:
classified = self.classifier(lstm_prediction)
final_lstm_predictions.append(F.expand_dims(classified,
axis=0))
final_lstm_predictions = F.concat(final_lstm_predictions, axis=0)
overall_predictions.append(final_lstm_predictions)
return overall_predictions, rois, points
def __call__(self, x):
return F.max_pooling_2d(x, self.ksize, self.stride, self.pad, self.cover_all, self.use_cudnn)
def __call__(self, x_data, y_data, train=True, n_patches=32):
if not isinstance(x_data, Variable):
length = x_data.shape[0]
x1 = Variable(x_data[0:length:2])
x2 = Variable(x_data[1:length:2])
else:
x = x_data
x_data = x.data
self.n_images = 1
self.n_patches = x_data.shape[0] / 2
self.n_patches_per_image = self.n_patches / self.n_images
h = F.relu(self.conv1(x1))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv3(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv5(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.conv7(h))
h = F.max_pooling_2d(h, 2)
h = F.relu(self.convP1(h))
h = F.max_pooling_2d(h, int(x_data.shape[2] / 8))
########## gradient stream
h1 = F.relu(self.conv2_1(x2))
h1 = F.max_pooling_2d(h1, 2)
h1 = F.relu(self.conv2_3(h1))
h1 = F.max_pooling_2d(h1, 2)
h1 = F.relu(self.conv2_5(h1))
h1 = F.max_pooling_2d(h1, 2)
h1 = F.relu(self.conv2_7(h1))
h1 = F.max_pooling_2d(h1, 2)
h1 = F.relu(self.convP2(h1))
h1 = F.max_pooling_2d(h1, int(x_data.shape[2] / 8))
#
h = F.concat((h, h1), axis=1)
h_ = h
self.h = h_
h = F.dropout(F.relu(self.fc1(h_)), ratio=0.5)
h = self.fc2(h)
#########################
a_patch = xp.ones_like(h.data)
t = xp.repeat(y_data[0:length:2], 1)
t_patch = xp.array(t.astype(np.float32))
self.average_loss(h, a_patch, t_patch)
model = self
self._save_model(model)
if train:
reporter.report({'loss': self.loss}, self)
# print 'self.lose:', self.loss.data
return self.loss
else:
return self.loss, self.y
def __call__(self, x):
return functions.max_pooling_2d(x, self.ksize, self.stride, self.pad)
def encode(self, in_data):
image = in_data['image']
keypoints = in_data['keypoints']
bbox = in_data['bbox']
is_labeled = in_data['is_labeled']
dataset_type = in_data['dataset_type']
inW, inH = self.insize
outW, outH = self.outsize
gridW, gridH = self.gridsize
K = len(self.keypoint_names)
delta = np.zeros((K, outH, outW), dtype=np.float32)
tx = np.zeros((K, outH, outW), dtype=np.float32)
ty = np.zeros((K, outH, outW), dtype=np.float32)
tw = np.zeros((K, outH, outW), dtype=np.float32)
th = np.zeros((K, outH, outW), dtype=np.float32)
te = np.zeros((len(self.edges), self.local_grid_size[1],
self.local_grid_size[0], outH, outW),
dtype=np.float32)
# Set delta^i_k
for (x, y, w, h), points, labeled in zip(bbox, keypoints, is_labeled):
if dataset_type == 'mpii':
partsW, partsH = self.parts_scale * math.sqrt(w * w + h * h)
instanceW, instanceH = self.instance_scale * math.sqrt(w * w +
h * h)
elif dataset_type == 'coco':
partsW, partsH = self.parts_scale * math.sqrt(w * w + h * h)
instanceW, instanceH = w, h
else:
raise ValueError(
"must be 'mpii' or 'coco' but actual {}".format(
dataset_type))
cy = y + h / 2
cx = x + w / 2
points = [[cy, cx]] + list(points)
labeled = [True] + list(labeled)
for k, (yx, l) in enumerate(zip(points, labeled)):
if not l:
continue
cy = yx[0] / gridH
cx = yx[1] / gridW
ix, iy = int(cx), int(cy)
sizeW = instanceW if k == 0 else partsW
sizeH = instanceH if k == 0 else partsH
if 0 <= iy < outH and 0 <= ix < outW:
delta[k, iy, ix] = 1
tx[k, iy, ix] = cx - ix
ty[k, iy, ix] = cy - iy
tw[k, iy, ix] = sizeW / inW
th[k, iy, ix] = sizeH / inH
for ei, (s, t) in enumerate(self.edges):
if not labeled[s]:
continue
if not labeled[t]:
continue
src_yx = points[s]
tar_yx = points[t]
iyx = (int(src_yx[0] / gridH), int(src_yx[1] / gridW))
jyx = (int(tar_yx[0] / gridH) - iyx[0] +
self.local_grid_size[1] // 2, int(tar_yx[1] / gridW) -
iyx[1] + self.local_grid_size[0] // 2)
if iyx[0] < 0 or iyx[1] < 0 or iyx[0] >= outH or iyx[1] >= outW:
continue
if jyx[0] < 0 or jyx[1] < 0 or jyx[0] >= self.local_grid_size[
1] or jyx[1] >= self.local_grid_size[0]:
continue
te[ei, jyx[0], jyx[1], iyx[0], iyx[1]] = 1
# define max(delta^i_k1, delta^j_k2) which is used for loss_limb
max_delta_ij = np.ones(
(len(self.edges), outH, outW, self.local_grid_size[1],
self.local_grid_size[0]),
dtype=np.float32)
or_delta = np.zeros((len(self.edges), outH, outW), dtype=np.float32)
for ei, (s, t) in enumerate(self.edges):
or_delta[ei] = np.minimum(delta[s] + delta[t], 1)
mask = F.max_pooling_2d(np.expand_dims(or_delta, axis=0),
ksize=(self.local_grid_size[1],
self.local_grid_size[0]),
stride=1,
pad=(self.local_grid_size[1] // 2,
self.local_grid_size[0] // 2))
mask = np.squeeze(mask.array, axis=0)
for index, _ in np.ndenumerate(mask):
max_delta_ij[index] *= mask[index]
max_delta_ij = max_delta_ij.transpose(0, 3, 4, 1, 2)
# preprocess image
image = self.feature_layer.prepare(image)
return image, delta, max_delta_ij, tx, ty, tw, th, te
def __call__(self, x):
h1 = F.max_pooling_2d(F.relu(self.cn1(x)), 2)
h2 = F.max_pooling_2d(F.relu(self.cn2(h1)), 2)
h3 = F.dropout(F.relu(self.fc1(h2)))
return self.fc2(h3)
def extract_pool11(self, x_vis):
x1 = Variable(x_vis.reshape(1, 1, x_vis.shape[0], x_vis.shape[1]),
volatile=True)
h1 = F.max_pooling_2d(F.relu(self.bn11(self.conv11(x1))), 2, stride=2)
return h1
def calc(self, x):
if self.bn:
h1_1 = F.relu(self.bnc1_1(self.c1_1(x)))
h1_2 = F.relu(self.bnc1_2(self.c1_2(h1_1)))
# p1 = F.max_pooling_2d(h1_2, ksize=2, stride=2)
h2_1 = F.relu(self.bnc2_1(self.c2_1(h1_2)))
h2_2 = F.relu(self.bnc2_2(self.c2_2(h2_1)))
#p2 = F.max_pooling_2d(h2_2, ksize=2, stride=2)
del h2_1
h3_1 = F.relu(self.bndi1_1(self.di1_1(h2_2)))
h3_2 = F.relu(self.bndi2_1(self.di2_1(h3_1)))
h3_3 = F.relu(self.bndi3_1(self.di3_1(h3_2)))
#p3 = F.max_pooling_2d(h3_2, ksize=2, stride=2)
del h3_1
#up4 = F.relu(self.bnup4(self.up4(h5_2)))
#up4 = self.__crop_to_target_2d(up4, h4_2)
dh4_1 = F.relu(self.bnd4_1(self.dc4_1(F.concat([h3_3, h2_2]))))
dh4_2 = F.relu(self.bnd4_2(self.dc4_2(dh4_1)))
dh3_1 = F.relu(self.bnd3_1(self.dc3_1(F.concat([dh4_2, h1_2]))))
#dh3_2 = F.relu(self.bnd3_2(self.dc3_2(dh3_1)))
del dh4_2, dh4_1
score = self.score(dh3_1)
#del dh1_2
else:
h1_1 = F.relu(self.c1_1(x))
h1_2 = F.relu(self.c1_2(h1_1))
p1 = F.max_pooling_2d(h1_2, ksize=2, stride=2)
del h1_1
h2_1 = F.relu(self.c2_1(p1))
h2_2 = F.relu(self.c2_2(h2_1))
p2 = F.max_pooling_2d(h2_2, ksize=2, stride=2)
del p1, h1_1
h3_1 = F.relu(self.c3_1(p2))
h3_2 = F.relu(self.c3_2(h3_1))
p3 = F.max_pooling_2d(h3_2, ksize=2, stride=2)
del p2, h3_1
h4_1 = F.relu(self.c4_1(p3))
h4_2 = F.relu(self.c4_2(h4_1))
p4 = F.max_pooling_2d(h4_2, ksize=2, stride=2)
del p3, h4_1
h5_1 = F.relu(self.c5_1(p4))
h5_2 = F.relu(self.c5_2(h5_1))
del p4, h5_1
up4 = F.relu(self.up4(h5_2))
up4 = self.__crop_to_target_2d(up4, h4_2)
dh4_1 = F.relu(self.dc4_1(F.concat([h4_2, up4])))
dh4_2 = F.relu(self.dc4_2(dh4_1))
del h5_2, up4, h4_2, dh4_1
up3 = F.relu(self.up3(dh4_2))
up3 = self.__crop_to_target_2d(up3, h3_2)
dh3_1 = F.relu(self.dc3_1(F.concat([h3_2, up3])))
dh3_2 = F.relu(self.dc3_2(dh3_1))
del dh4_2, up3, h3_2, dh3_1
up2 = F.relu(self.up2(dh3_2))
up2 = self.__crop_to_target_2d(up2, h2_2)
dh2_1 = F.relu(self.dc2_1(F.concat([h2_2, up2])))
dh2_2 = F.relu(self.dc2_2(dh2_1))
del dh3_2, up2, h2_2, dh2_1
up1 = F.relu(self.up1(dh2_2))
up1 = self.__crop_to_target_2d(up1, h1_2)
dh1_1 = F.relu(self.dc1_1(F.concat([h1_2, up1])))
dh1_2 = F.relu(self.dc1_2(dh1_1))
del dh2_2, up1, h1_2, dh1_1
score = self.score(dh1_2)
del dh1_2
return score
def __call__(self, x, t):
h = cr(self.conv_d0a_b, x)
assert h.shape[1:] == (64, 570, 570)
d0c = cr(self.conv_d0b_c, h)
assert d0c.shape[1:] == (64, 568, 568)
h = F.max_pooling_2d(d0c, ksize=2, stride=2, pad=0)
assert h.shape[1:] == (64, 284, 284)
h = cr(self.conv_d1a_b, h)
assert h.shape[1:] == (128, 282, 282)
d1c = cr(self.conv_d1b_c, h)
assert d1c.shape[1:] == (128, 280, 280)
h = F.max_pooling_2d(d1c, ksize=2, stride=2, pad=0)
assert h.shape[1:] == (128, 140, 140)
h = cr(self.conv_d2a_b, h)
assert h.shape[1:] == (256, 138, 138)
d2c = cr(self.conv_d2b_c, h)
assert d2c.shape[1:] == (256, 136, 136)
h = F.max_pooling_2d(d2c, ksize=2, stride=2, pad=0)
assert h.shape[1:] == (256, 68, 68)
h = cr(self.conv_d3a_b, h)
assert h.shape[1:] == (512, 66, 66)
h = cr(self.conv_d3b_c, h)
assert h.shape[1:] == (512, 64, 64)
d3c = F.dropout(h, 0.5)
h = F.max_pooling_2d(d3c, ksize=2, stride=2, pad=0)
assert h.shape[1:] == (512, 32, 32)
h = cr(self.conv_d4a_b, h)
assert h.shape[1:] == (1024, 30, 30)
h = cr(self.conv_d4b_c, h)
assert h.shape[1:] == (1024, 28, 28)
d4c = F.dropout(h, 0.5)
h = cr(self.upconv_d4c_u3a, d4c)
assert h.shape[1:] == (512, 56, 56)
h = F.concat([d3c[:, :, 4:-4, 4:-4], h])
assert h.shape[1:] == (1024, 56, 56)
h = cr(self.conv_u3b_c, h)
assert h.shape[1:] == (512, 54, 54)
h = cr(self.conv_u3c_d, h)
assert h.shape[1:] == (512, 52, 52)
h = cr(self.upconv_u3d_u2a, h)
assert h.shape[1:] == (256, 104, 104)
h = F.concat([d2c[:, :, 16:-16, 16:-16], h])
assert h.shape[1:] == (512, 104, 104)
h = cr(self.conv_u2b_c, h)
assert h.shape[1:] == (256, 102, 102)
h = cr(self.conv_u2c_d, h)
assert h.shape[1:] == (256, 100, 100)
h = cr(self.upconv_u2d_u1a, h)
assert h.shape[1:] == (128, 200, 200)
h = F.concat([d1c[:, :, 40:-40, 40:-40], h])
assert h.shape[1:] == (256, 200, 200)
h = cr(self.conv_u1b_c, h)
assert h.shape[1:] == (128, 198, 198)
h = cr(self.conv_u1c_d, h)
assert h.shape[1:] == (128, 196, 196)
h = cr(self.upconv_u1d_u0a, h)
assert h.shape[1:] == (128, 392, 392)
h = F.concat([d0c[:, :, 88:-88, 88:-88], h])
assert h.shape[1:] == (192, 392, 392)
h = cr(self.conv_u0b_c, h)
assert h.shape[1:] == (64, 390, 390)
h = cr(self.conv_u0c_d, h)
assert h.shape[1:] == (64, 388, 388)
h = self.conv_u0d_score(h)
assert h.shape[1:] == (2, 388, 388)
return F.softmax_cross_entropy(h, t)
