def test_catch_unconnected(self):
"""
Catch mapping spatially unconnected tops.
"""
n = coord_net_spec()
n.ip = L.InnerProduct(n.deconv, num_output=10)
with self.assertRaises(RuntimeError):
coord_map_from_to(n.ip, n.data)
def test_catch_unconnected(self):
"""
Catch mapping spatially unconnected tops.
"""
n = coord_net_spec()
n.ip = L.InnerProduct(n.deconv, num_output=10)
with self.assertRaises(RuntimeError):
coord_map_from_to(n.ip, n.data)
def test_crop_of_crop(self):
"""
Map coordinates through Crop layer:
crop an already-cropped output to the input and check change in offset.
"""
n = coord_net_spec()
offset = random.randint(0, 10)
ax, a, b = coord_map_from_to(n.deconv, n.data)
n.crop = L.Crop(n.deconv, n.data, axis=2, offset=offset)
ax_crop, a_crop, b_crop = coord_map_from_to(n.crop, n.data)
self.assertEquals(ax, ax_crop)
self.assertEquals(a, a_crop)
self.assertEquals(b + offset, b_crop)
def test_crop_of_crop(self):
"""
Map coordinates through Crop layer:
crop an already-cropped output to the input and check change in offset.
"""
n = coord_net_spec()
offset = random.randint(0, 10)
ax, a, b = coord_map_from_to(n.deconv, n.data)
n.crop = L.Crop(n.deconv, n.data, axis=2, offset=offset)
ax_crop, a_crop, b_crop = coord_map_from_to(n.crop, n.data)
self.assertEquals(ax, ax_crop)
self.assertEquals(a, a_crop)
self.assertEquals(b + offset, b_crop)
def test_multi_conv(self):
"""
Multiple bottoms/tops of a layer are identically mapped.
"""
n = coord_net_spec()
# multi bottom/top
n.conv_data, n.conv_aux = L.Convolution(
n.data, n.aux, ntop=2, num_output=10, kernel_size=5, stride=2,
pad=0)
ax1, a1, b1 = coord_map_from_to(n.conv_data, n.data)
ax2, a2, b2 = coord_map_from_to(n.conv_aux, n.aux)
self.assertEquals(ax1, ax2)
self.assertEquals(a1, a2)
self.assertEquals(b1, b2)
def test_multi_conv(self):
"""
Multiple bottoms/tops of a layer are identically mapped.
"""
n = coord_net_spec()
# multi bottom/top
n.conv_data, n.conv_aux = L.Convolution(
n.data, n.aux, ntop=2, num_output=10, kernel_size=5, stride=2,
pad=0)
ax1, a1, b1 = coord_map_from_to(n.conv_data, n.data)
ax2, a2, b2 = coord_map_from_to(n.conv_aux, n.aux)
self.assertEquals(ax1, ax2)
self.assertEquals(a1, a2)
self.assertEquals(b1, b2)
def test_rect(self):
"""
Anisotropic mapping is equivalent to its isotropic parts.
"""
n3x3 = coord_net_spec(ks=3, stride=1, pad=0)
n5x5 = coord_net_spec(ks=5, stride=2, pad=10)
n3x5 = coord_net_spec(ks=[3, 5], stride=[1, 2], pad=[0, 10])
ax_3x3, a_3x3, b_3x3 = coord_map_from_to(n3x3.deconv, n3x3.data)
ax_5x5, a_5x5, b_5x5 = coord_map_from_to(n5x5.deconv, n5x5.data)
ax_3x5, a_3x5, b_3x5 = coord_map_from_to(n3x5.deconv, n3x5.data)
self.assertTrue(ax_3x3 == ax_5x5 == ax_3x5)
self.assertEquals(a_3x3, a_3x5[0])
self.assertEquals(b_3x3, b_3x5[0])
self.assertEquals(a_5x5, a_3x5[1])
self.assertEquals(b_5x5, b_3x5[1])
def test_pass(self):
"""
A pass-through layer (ReLU) and conv (1x1, stride 1, pad 0)
both do identity mapping.
"""
n = coord_net_spec()
ax, a, b = coord_map_from_to(n.deconv, n.data)
n.relu = L.ReLU(n.deconv)
n.conv1x1 = L.Convolution(
n.relu, num_output=10, kernel_size=1, stride=1, pad=0)
for top in [n.relu, n.conv1x1]:
ax_pass, a_pass, b_pass = coord_map_from_to(top, n.data)
self.assertEquals(ax, ax_pass)
self.assertEquals(a, a_pass)
self.assertEquals(b, b_pass)
def test_rect(self):
"""
Anisotropic mapping is equivalent to its isotropic parts.
"""
n3x3 = coord_net_spec(ks=3, stride=1, pad=0)
n5x5 = coord_net_spec(ks=5, stride=2, pad=10)
n3x5 = coord_net_spec(ks=[3, 5], stride=[1, 2], pad=[0, 10])
ax_3x3, a_3x3, b_3x3 = coord_map_from_to(n3x3.deconv, n3x3.data)
ax_5x5, a_5x5, b_5x5 = coord_map_from_to(n5x5.deconv, n5x5.data)
ax_3x5, a_3x5, b_3x5 = coord_map_from_to(n3x5.deconv, n3x5.data)
self.assertTrue(ax_3x3 == ax_5x5 == ax_3x5)
self.assertEquals(a_3x3, a_3x5[0])
self.assertEquals(b_3x3, b_3x5[0])
self.assertEquals(a_5x5, a_3x5[1])
self.assertEquals(b_5x5, b_3x5[1])
def test_pass(self):
"""
A pass-through layer (ReLU) and conv (1x1, stride 1, pad 0)
both do identity mapping.
"""
n = coord_net_spec()
ax, a, b = coord_map_from_to(n.deconv, n.data)
n.relu = L.ReLU(n.deconv)
n.conv1x1 = L.Convolution(
n.relu, num_output=10, kernel_size=1, stride=1, pad=0)
for top in [n.relu, n.conv1x1]:
ax_pass, a_pass, b_pass = coord_map_from_to(top, n.data)
self.assertEquals(ax, ax_pass)
self.assertEquals(a, a_pass)
self.assertEquals(b, b_pass)
def test_nd_conv(self):
"""
ND conv maps the same way in more dimensions.
"""
n = caffe.NetSpec()
# define data with 3 spatial dimensions, otherwise the same net
n.data = L.Input(shape=dict(dim=[2, 3, 100, 100, 100]))
n.conv = L.Convolution(n.data,
num_output=10,
kernel_size=[3, 3, 3],
stride=[1, 1, 1],
pad=[0, 1, 2])
n.pool = L.Pooling(n.conv,
pool=P.Pooling.MAX,
kernel_size=2,
stride=2,
pad=0)
n.deconv = L.Deconvolution(n.pool,
num_output=10,
kernel_size=4,
stride=2,
pad=0)
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertTrue(len(a) == len(b))
self.assertTrue(np.all(a == 1))
self.assertEquals(b[0] - 1, b[1])
self.assertEquals(b[1] - 1, b[2])
def test_conv_pool_deconv(self):
"""
Map through conv, pool, and deconv.
"""
n = coord_net_spec()
# identity for 2x pool, 2x deconv
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertEquals(a, 1)
self.assertEquals(b, 0)
# shift-by-one for 4x pool, 4x deconv
n = coord_net_spec(pool=4, dstride=4)
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertEquals(a, 1)
self.assertEquals(b, -1)
def test_conv_pool_deconv(self):
"""
Map through conv, pool, and deconv.
"""
n = coord_net_spec()
# identity for 2x pool, 2x deconv
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertEquals(a, 1)
self.assertEquals(b, 0)
# shift-by-one for 4x pool, 4x deconv
n = coord_net_spec(pool=4, dstride=4)
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertEquals(a, 1)
self.assertEquals(b, -1)
def test_padding(self):
"""
Padding conv adds offset while padding deconv subtracts offset.
"""
n = coord_net_spec()
ax, a, b = coord_map_from_to(n.deconv, n.data)
pad = random.randint(0, 10)
# conv padding
n = coord_net_spec(pad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b - pad, b_pad)
# deconv padding
n = coord_net_spec(dpad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b + pad, b_pad)
# pad both to cancel out
n = coord_net_spec(pad=pad, dpad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b, b_pad)
def test_padding(self):
"""
Padding conv adds offset while padding deconv subtracts offset.
"""
n = coord_net_spec()
ax, a, b = coord_map_from_to(n.deconv, n.data)
pad = random.randint(0, 10)
# conv padding
n = coord_net_spec(pad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b - pad, b_pad)
# deconv padding
n = coord_net_spec(dpad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b + pad, b_pad)
# pad both to cancel out
n = coord_net_spec(pad=pad, dpad=pad)
_, a_pad, b_pad = coord_map_from_to(n.deconv, n.data)
self.assertEquals(a, a_pad)
self.assertEquals(b, b_pad)
def test_nd_conv(self):
"""
ND conv maps the same way in more dimensions.
"""
n = caffe.NetSpec()
# define data with 3 spatial dimensions, otherwise the same net
n.data = L.Input(shape=dict(dim=[2, 3, 100, 100, 100]))
n.conv = L.Convolution(
n.data, num_output=10, kernel_size=[3, 3, 3], stride=[1, 1, 1],
pad=[0, 1, 2])
n.pool = L.Pooling(
n.conv, pool=P.Pooling.MAX, kernel_size=2, stride=2, pad=0)
n.deconv = L.Deconvolution(
n.pool, num_output=10, kernel_size=4, stride=2, pad=0)
ax, a, b = coord_map_from_to(n.deconv, n.data)
self.assertEquals(ax, 1)
self.assertTrue(len(a) == len(b))
self.assertTrue(np.all(a == 1))
self.assertEquals(b[0] - 1, b[1])
self.assertEquals(b[1] - 1, b[2])
def simple_net(split, initialize_fc8=False, cur_shape = None, next_shape = None, batch_size=1, num_threads=1, max_queue_size=5):
#Get crop layer parameters
tmp_net = caffe.NetSpec()
tmp_net.im, tmp_net.label = L.MemoryData(batch_size=1, channels=3, height=244, width=244, ntop=2)
conv_vgg(tmp_net, tmp_net.im, suffix='', last_layer_pad=0, first_layer_pad=100)
tmp_net.fc6, tmp_net.relu6 = conv_relu(tmp_net.conv5_3, 4096, ks=7, dilation=4)
tmp_net.fc7, tmp_net.relu7 = conv_relu(tmp_net.relu6, 4096, ks=1, pad=0)
tmp_net.fc8 = L.Convolution(tmp_net.relu7, kernel_size=1, num_output=2)
tmp_net.upscore = L.Deconvolution(tmp_net.fc8, convolution_param=dict(kernel_size=16, stride=8, num_output=2))
ax, a, b = coord_map_from_to(tmp_net.upscore, tmp_net.im)
assert (a == 1).all(), 'scale mismatch on crop (a = {})'.format(a)
assert (b <= 0).all(), 'cannot crop negative offset (b = {})'.format(b)
assert (np.round(b) == b).all(), 'cannot crop noninteger offset (b = {})'.format(b)
#
#Create network
n = caffe.NetSpec()
if split == 'train':
pydata_params = dict(batch_size=batch_size, im_shape=tuple(next_shape), num_threads=num_threads, max_queue_size=max_queue_size)
n.cur_im, n.masked_im, n.next_im, n.label = L.Python(module='coco_transformed_datalayers_prefetch', layer='CocoTransformedDataLayerPrefetch', ntop=4, param_str=str(pydata_params))
elif split == 'val':
pydata_params = dict(batch_size=batch_size, im_shape=tuple(next_shape), num_threads=num_threads, max_queue_size=max_queue_size)
n.cur_im, n.masked_im, n.next_im, n.label = L.Python(module='coco_transformed_datalayers_prefetch', layer='CocoTransformedDataLayerPrefetch', ntop=4, param_str=str(pydata_params))
elif split == 'deploy':
n.cur_im, n.label_1 = L.MemoryData(batch_size=1, channels=3, height=244, width=244, ntop=2)
n.masked_im, n.label_2 = L.MemoryData(batch_size=1, channels=3, height=244, width=244, ntop=2)
n.next_im, n.label_3 = L.MemoryData(batch_size=1, channels=3, height=244, width=244, ntop=2)
else:
raise Exception
if cur_shape is None or next_shape is None:
concat_pad = np.zeros((2,))
else:
concat_pad = (next_shape - cur_shape)/2.0/8.0
if not all(concat_pad == np.round(concat_pad)):
raise Exception
conv_vgg(n, n.cur_im, suffix='c', last_layer_pad=concat_pad, first_layer_pad=100)
conv_vgg(n, n.masked_im, suffix='m', last_layer_pad=concat_pad, first_layer_pad=100)
conv_vgg(n, n.next_im, suffix='n', last_layer_pad=0, first_layer_pad=100)
# concatination
n.concat1 = L.Concat(n.relu5_3c, n.relu5_3m, n.relu5_3n)
# fully conv
n.fc6, n.relu6 = conv_relu(n.concat1, 4096, ks=7, dilation=4)
if split == 'train':
n.drop6 = L.Dropout(n.relu6, dropout_ratio=0.5, in_place=True)
n.fc7, n.relu7 = conv_relu(n.drop6, 4096, ks=1, pad=0)
n.drop7 = L.Dropout(n.relu7, dropout_ratio=0.5, in_place=True)
n.fc8 = L.Convolution(n.drop7, kernel_size=1, param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)], num_output=2)
else:
n.fc7, n.relu7 = conv_relu(n.relu6, 4096, ks=1, pad=0)
if initialize_fc8:
n.fc8 = L.Convolution(n.relu7, kernel_size=1, param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)], weight_filler=dict(type='gaussian', std=.01), num_output=2)
else:
n.fc8 = L.Convolution(n.relu7, kernel_size=1, param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)], num_output=2)
n.upscore = L.Deconvolution(n.fc8, convolution_param=dict(kernel_size=16, stride=8, num_output=2, group=2, weight_filler=dict(type='bilinear'),
bias_term=False), param=dict(lr_mult=0, decay_mult=0))
n.score = L.Crop(n.upscore, n.next_im,
crop_param=dict(axis=ax + 1, # +1 for first cropping dim.
offset=list(-np.round(b).astype(int))))
if split != 'deploy':
n.loss = L.SoftmaxWithLoss(n.score, n.label,
loss_param=dict(ignore_label=255))
else:
n.prop = L.Softmax(n.score)
return n
def simple_net(split,
initialize_fc8=False,
cur_shape=None,
next_shape=None,
batch_size=1,
num_threads=1,
max_queue_size=5):
#Get crop layer parameters
tmp_net = caffe.NetSpec()
tmp_net.im, tmp_net.label = L.MemoryData(batch_size=1,
channels=3,
height=244,
width=244,
ntop=2)
conv_vgg(tmp_net,
tmp_net.im,
suffix='',
last_layer_pad=0,
first_layer_pad=100)
tmp_net.fc6, tmp_net.relu6 = conv_relu(tmp_net.conv5_3,
4096,
ks=7,
dilation=4)
tmp_net.fc7, tmp_net.relu7 = conv_relu(tmp_net.relu6, 4096, ks=1, pad=0)
tmp_net.fc8 = L.Convolution(tmp_net.relu7, kernel_size=1, num_output=2)
tmp_net.upscore = L.Deconvolution(tmp_net.fc8,
convolution_param=dict(kernel_size=16,
stride=8,
num_output=2))
ax, a, b = coord_map_from_to(tmp_net.upscore, tmp_net.im)
assert (a == 1).all(), 'scale mismatch on crop (a = {})'.format(a)
assert (b <= 0).all(), 'cannot crop negative offset (b = {})'.format(b)
assert (np.round(b) == b
).all(), 'cannot crop noninteger offset (b = {})'.format(b)
#
#Create network
n = caffe.NetSpec()
if split == 'train':
pydata_params = dict(batch_size=batch_size,
im_shape=tuple(next_shape),
num_threads=num_threads,
max_queue_size=max_queue_size)
n.cur_im, n.masked_im, n.next_im, n.label = L.Python(
module='coco_transformed_datalayers_prefetch',
layer='CocoTransformedDataLayerPrefetch',
ntop=4,
param_str=str(pydata_params))
elif split == 'val':
pydata_params = dict(batch_size=batch_size,
im_shape=tuple(next_shape),
num_threads=num_threads,
max_queue_size=max_queue_size)
n.cur_im, n.masked_im, n.next_im, n.label = L.Python(
module='coco_transformed_datalayers_prefetch',
layer='CocoTransformedDataLayerPrefetch',
ntop=4,
param_str=str(pydata_params))
elif split == 'deploy':
n.cur_im, n.label_1 = L.MemoryData(batch_size=1,
channels=3,
height=244,
width=244,
ntop=2)
n.masked_im, n.label_2 = L.MemoryData(batch_size=1,
channels=3,
height=244,
width=244,
ntop=2)
n.next_im, n.label_3 = L.MemoryData(batch_size=1,
channels=3,
height=244,
width=244,
ntop=2)
else:
raise Exception
if cur_shape is None or next_shape is None:
concat_pad = np.zeros((2, ))
else:
concat_pad = (next_shape - cur_shape) / 2.0 / 8.0
if not all(concat_pad == np.round(concat_pad)):
raise Exception
conv_vgg(n,
n.cur_im,
suffix='c',
last_layer_pad=concat_pad,
first_layer_pad=100)
conv_vgg(n,
n.masked_im,
suffix='m',
last_layer_pad=concat_pad,
first_layer_pad=100)
conv_vgg(n, n.next_im, suffix='n', last_layer_pad=0, first_layer_pad=100)
# concatination
n.concat1 = L.Concat(n.relu5_3c, n.relu5_3m, n.relu5_3n)
# fully conv
n.fc6, n.relu6 = conv_relu(n.concat1, 4096, ks=7, dilation=4)
if split == 'train':
n.drop6 = L.Dropout(n.relu6, dropout_ratio=0.5, in_place=True)
n.fc7, n.relu7 = conv_relu(n.drop6, 4096, ks=1, pad=0)
n.drop7 = L.Dropout(n.relu7, dropout_ratio=0.5, in_place=True)
n.fc8 = L.Convolution(n.drop7,
kernel_size=1,
param=[
dict(lr_mult=1, decay_mult=1),
dict(lr_mult=2, decay_mult=0)
],
num_output=2)
else:
n.fc7, n.relu7 = conv_relu(n.relu6, 4096, ks=1, pad=0)
if initialize_fc8:
n.fc8 = L.Convolution(n.relu7,
kernel_size=1,
param=[
dict(lr_mult=1, decay_mult=1),
dict(lr_mult=2, decay_mult=0)
],
weight_filler=dict(type='gaussian', std=.01),
num_output=2)
else:
n.fc8 = L.Convolution(n.relu7,
kernel_size=1,
param=[
dict(lr_mult=1, decay_mult=1),
dict(lr_mult=2, decay_mult=0)
],
num_output=2)
n.upscore = L.Deconvolution(n.fc8,
convolution_param=dict(
kernel_size=16,
stride=8,
num_output=2,
group=2,
weight_filler=dict(type='bilinear'),
bias_term=False),
param=dict(lr_mult=0, decay_mult=0))
n.score = L.Crop(
n.upscore,
n.next_im,
crop_param=dict(
axis=ax + 1, # +1 for first cropping dim.
offset=list(-np.round(b).astype(int))))
if split != 'deploy':
n.loss = L.SoftmaxWithLoss(n.score,
n.label,
loss_param=dict(ignore_label=255))
else:
n.prop = L.Softmax(n.score)
return n
