def box_overlap_opr(box, gt):
"""
Compute the overlaps between box and gt(_box)
box: (N, 4) Tensor
gt : (K, 4) Tensor
return: (N, K) Tensor, stores Max(0, intersection/union)
"""
N = box.shape[0]
K = gt.shape[0]
target_shape = (N, K, 4)
b_box = box.add_axis(1).broadcast(target_shape)
b_gt = gt.add_axis(0).broadcast(target_shape)
iw = (
Min(b_box[:, :, 2], b_gt[:, :, 2]) - \
Max(b_box[:, :, 0], b_gt[:, :, 0]) + 1)
ih = (
Min(b_box[:, :, 3], b_gt[:, :, 3]) - \
Max(b_box[:, :, 1], b_gt[:, :, 1]) + 1)
inter = Max(iw, 0) * Max(ih, 0)
# Use the broadcast to save some time
area_box = (box[:, 2] - box[:, 0] + 1) * (box[:, 3] - box[:, 1] + 1)
area_gt = (gt[:, 2] - gt[:, 0] + 1) * (gt[:, 3] - gt[:, 1] + 1)
area_target_shape = (N, K)
b_area_box = area_box.add_axis(1).broadcast(area_target_shape)
b_area_gt = area_gt.add_axis(0).broadcast(area_target_shape)
union = b_area_box + b_area_gt - inter
overlaps = Max(inter / union, 0)
return overlaps
def box_overlap_ignore_opr(box, gt, *, ignore_label=-1, return_separate=False):
"""
Compute the overlaps between box and gt(_box)
box: (N, 4) Tensor
gt : (K, 5) Tensor, the last col shows the labels of gt
return: (N, K) Tensor, stores Max(0, intersection/union)
Here, we consider the ignore_label of gt boxes. When compute
box vs ignored_gt, the overlap is replaced by inter / box_area.
This operation will force the boxes near to ignore gt_boxes to
be matched to ignored boxes rather than fg or bg labels.
"""
N = box.shape[0]
K = gt.shape[0]
target_shape = (N, K, 4)
b_box = box.add_axis(1).broadcast(target_shape)
b_gt = gt[:, :4].add_axis(0).broadcast(target_shape)
# intersection of boxes
iw = (Min(b_box[:, :, 2], b_gt[:, :, 2]) - \
Max(b_box[:, :, 0], b_gt[:, :, 0]) + 1)
ih = (Min(b_box[:, :, 3], b_gt[:, :, 3]) - \
Max(b_box[:, :, 1], b_gt[:, :, 1]) + 1)
inter = Max(iw, 0) * Max(ih, 0)
# Use the broadcast to save some time
area_box = (box[:, 2] - box[:, 0] + 1) * (box[:, 3] - box[:, 1] + 1)
area_gt = (gt[:, 2] - gt[:, 0] + 1) * (gt[:, 3] - gt[:, 1] + 1)
area_target_shape = (N, K)
b_area_box = area_box.add_axis(1).broadcast(area_target_shape)
b_area_gt = area_gt.add_axis(0).broadcast(area_target_shape)
union = b_area_box + b_area_gt - inter
overlaps_normal = Max(inter / union, 0)
overlaps_ignore = Max(inter / b_area_box, 0)
gt_ignore_mask = gt[:, 4].eq(ignore_label).add_axis(0).broadcast(
area_target_shape)
overlaps_normal *= (1 - gt_ignore_mask)
overlaps_ignore *= gt_ignore_mask
if return_separate:
return overlaps_normal, overlaps_ignore
else:
overlaps = overlaps_normal + overlaps_ignore
return overlaps
def clip_boxes_opr(boxes, im_info):
""" Clip the boxes into the image region."""
# x1 >=0
box_x1 = Max(Min(boxes[:, 0::4], im_info[1] - 1), 0)
# y1 >=0
box_y1 = Max(Min(boxes[:, 1::4], im_info[0] - 1), 0)
# x2 < im_info[1]
box_x2 = Max(Min(boxes[:, 2::4], im_info[1] - 1), 0)
# y2 < im_info[0]
box_y2 = Max(Min(boxes[:, 3::4], im_info[0] - 1), 0)
# clip_box = Concat([box_x1, box_y1, box_x2, box_y2], axis=1)
clip_box = _concat_new_axis(box_x1, box_y1, box_x2, box_y2, 2)\
.reshape(boxes.shape[0], -1)
return clip_box
def dfconv(inp, chl, isrelu, ker_shape = 3, stride = 1, padding = 1, dx = [-1, 0, 1], dy = [-1, 0, 1]):
inp = Conv2D(
name + "conv", inp, kernel_shape = 3, stride = 1, padding = 1,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = ((1) / (ker_shape**2 * inp.partial_shape[1]))**0.5),
nonlinearity = Identity()
)
inp = BN(name + "BN", inp, eps = 1e-9)
global idx
#idx += 1
gamma = 0.001
offsetx = inp.partial_shape[2] * Conv2D(
"conv{}_offsetx".format(idx + 1), inp, kernel_shape = ker_shape, stride = stride,
padding = padding,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = gamma / (ker_shape**2 * inp.partial_shape[2])),
nonlinearity = Identity()
)
offsety = inp.partial_shape[3] * Conv2D(
"conv{}_offsety".format(idx + 1), inp, kernel_shape = ker_shape, stride = stride,
padding = padding,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = gamma / (ker_shape**2 * inp.partial_shape[3])),
nonlinearity = Identity()
)
outputs = []
for sx in range(2):
for sy in range(2):
if sx == 0:
ofx = Floor(offsetx)
bilx = offsetx - ofx
else:
ofx = Ceil(offsetx)
bilx = ofx - offsetx
if sy == 0:
ofy = Floor(offsety)
bily = offsety - ofy
else:
ofy = Ceil(offsety)
bily = ofy - offsety
"""
No padding
padding1 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], 1, inp.partial_shape[3])))
padding2 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], inp.partial_shape[2] + 2, 1)))
arg_fea = Concat([padding1, inp, padding1], axis = 2)
arg_fea = Concat([padding2, arg_fea, padding2], axis = 3)
"""
arg_fea = inp
#one_mat = ConstProvider(np.ones((inp.partial_shape[2], inp.partial_shape[3])), dtype = np.int32)
one_mat = ConstProvider(1, dtype = np.int32).add_axis(0).broadcast((ofx.partial_shape[2], ofx.partial_shape[3]))
affx = (Cumsum(one_mat, axis = 0) - 1) * stride
affy = (Cumsum(one_mat, axis = 1) - 1) * stride
ofx = ofx + affx.dimshuffle('x', 'x', 0, 1)
ofy = ofy + affy.dimshuffle('x', 'x', 0, 1)
one_mat = ConstProvider(np.ones((ker_shape, ofx.partial_shape[2], ofx.partial_shape[3])))
#ofx[:, :ker_shape, :, :] -= 1
#ofx[:, ker_shape*2:, :, :] += 1
ofx += Concat([one_mat * i for i in dx], axis = 0).dimshuffle('x', 0, 1, 2)
#ofy[:, ::3, :, :] -= 1
#ofy[:, 2::3, :, :] += 1
one_mat = ones((1, ofx.partial_shape[2], ofx.partial_shape[3]))
one_mat = Concat([one_mat * i for i in dy], axis = 0)
one_mat = Concat([one_mat] * ker_shape, axis = 0)
ofy += one_mat.dimshuffle('x', 0, 1, 2)
ofx = Max(Min(ofx, arg_fea.partial_shape[2] - 1), 0)
ofy = Max(Min(ofy, arg_fea.partial_shape[3] - 1), 0)
def DeformReshape(inp, ker_shape):
inp = inp.reshape(inp.shape[0], ker_shape, ker_shape, inp.shape[2], inp.shape[3])
inp = inp.dimshuffle(0, 3, 1, 4, 2)
inp = inp.reshape(inp.shape[0], inp.shape[1] * inp.shape[2], inp.shape[3] * inp.shape[4])
return inp
ofx = DeformReshape(ofx, ker_shape)
ofy = DeformReshape(ofy, ker_shape)
bilx = DeformReshape(bilx, ker_shape)
bily = DeformReshape(bily, ker_shape)
of = ofx * arg_fea.shape[2] + ofy
arg_fea = arg_fea.reshape(arg_fea.shape[0], arg_fea.shape[1], -1)
of = of.reshape(ofx.shape[0], -1)
of = of.dimshuffle(0, 'x', 1)
#of = Concat([of] * arg_fea.partial_shape[1], axis = 1)
of = of.broadcast((of.shape[0], arg_fea.shape[1], of.shape[2]))
arx = Linspace(0, arg_fea.shape[0], arg_fea.shape[0], endpoint = False)
arx = arx.add_axis(1).add_axis(2).broadcast(of.shape)
ary = Linspace(0, arg_fea.shape[1], arg_fea.shape[1], endpoint = False)
ary = ary.add_axis(0).add_axis(2).broadcast(of.shape)
of = of.add_axis(3)
arx = arx.add_axis(3)
ary = ary.add_axis(3)
idxmap = Astype(Concat([arx, ary, of], axis = 3), np.int32)
"""
sample = []
for i in range(arg_fea.partial_shape[0]):
for j in range(arg_fea.partial_shape[1]):
sample.append(arg_fea[i][j].ai[of[i][j]].dimshuffle('x', 0))
sample = Concat(sample, axis = 0)
"""
sample = IndexingRemap(arg_fea, idxmap).reshape(inp.shape[0], inp.shape[1], bilx.shape[1], -1)
bilx = bilx.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
bily = bily.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
sample *= bilx * bily
outputs.append(sample)
output = outputs[0]
for i in outputs[1:]:
output += i
return conv_bn(output, ker_shape, 3, 0, chl, isrelu)
def dfpooling(name, inp, window = 2, padding = 0, dx = [0, 1], dy = [0, 1]):
#inp = ConstProvider([[[[1, 2], [3, 4]]]], dtype = np.float32)
"""
Add a new conv&bn to insure that the scale of the feature map is variance 1.
"""
ker_shape = window
stride = window
offsetlay = Conv2D(
name + "conv", inp, kernel_shape = 3, stride = 1, padding = 1,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = ((1) / (3**2 * inp.partial_shape[1]))**0.5),
nonlinearity = Identity()
)
#offsetlay = BN(name + "BN", offsetlay, eps = 1e-9)
offsetx = Conv2D(
name + "conv1x", offsetlay, kernel_shape = ker_shape, stride = stride,
padding = padding,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = (1 / (ker_shape**2 * inp.partial_shape[2]))**0.5),
nonlinearity = Identity()
)
offsety = Conv2D(
name + "conv1y", offsetlay, kernel_shape = ker_shape, stride = stride,
padding = padding,
output_nr_channel = ker_shape**2,
W = G(mean = 0, std = (1 / (ker_shape**2 * inp.partial_shape[3]))**0.5),
nonlinearity = Identity()
)
offset = Concat([offsetx, offsety], axis = 1)
ndim = ker_shape**2 * offsetx.partial_shape[2] * offsetx.partial_shape[3] * 2
offset = FullyConnected(
name + "offset", offsetx, output_dim = ndim,
W = G(mean = 0, std = (1 / ndim)**2),
#W = C(0),
b = C(0),
nonlinearity = Identity()
)
offsetx = offset[:, :ndim // 2].reshape(offsetx.shape)
offsety = offset[:, ndim // 2:].reshape(offsety.shape)
"""
offsetx = FullyConnected(
name + "offsetx", offsetx, output_dim = ndim,
W = G(mean = 0, std = gamma / ndim),
b = C(0),
nonlinearity = Identity()
)
offsetx = offsetx.reshape(offsety.shape)
offsety = FullyConnected(
name + "offsety", offsety, output_dim = ndim,
W = G(mean = 0, std = gamma / ndim),
b = C(0),
nonlinearity = Identity()
)
offsety = offsety.reshape(offsetx.shape)
print(offsety.partial_shape)
"""
#offsetx = ZeroGrad(offsetx)
#offsety = ZeroGrad(offsety)
outputs = []
for sx in range(2):
for sy in range(2):
if sx == 0:
ofx = Floor(offsetx)
bilx = 1 - (offsetx - ofx)
else:
ofx = Ceil(offsetx)
bilx = 1 - (ofx - offsetx)
if sy == 0:
ofy = Floor(offsety)
bily = 1 - (offsety - ofy)
else:
ofy = Ceil(offsety)
bily = 1 - (ofy - offsety)
"""
No padding
padding1 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], 1, inp.partial_shape[3])))
padding2 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], inp.partial_shape[2] + 2, 1)))
arg_fea = Concat([padding1, inp, padding1], axis = 2)
arg_fea = Concat([padding2, arg_fea, padding2], axis = 3)
"""
arg_fea = inp
#one_mat = ConstProvider(np.ones((inp.partial_shape[2], inp.partial_shape[3])), dtype = np.int32)
one_mat = ConstProvider(1, dtype = np.int32).add_axis(0).broadcast((ofx.shape[2], ofx.shape[3]))
affx = (Cumsum(one_mat, axis = 0) - 1) * stride
affy = (Cumsum(one_mat, axis = 1) - 1) * stride
ofx = ofx + affx.dimshuffle('x', 'x', 0, 1)
ofy = ofy + affy.dimshuffle('x', 'x', 0, 1)
one_mat = ConstProvider(np.ones((ker_shape, ofx.partial_shape[2], ofx.partial_shape[3])))
#ofx[:, :ker_shape, :, :] -= 1
#ofx[:, ker_shape*2:, :, :] += 1
ofx += Concat([one_mat * i for i in dx], axis = 0).dimshuffle('x', 0, 1, 2)
#ofy[:, ::3, :, :] -= 1
#ofy[:, 2::3, :, :] += 1
one_mat = ones((1, ofx.partial_shape[2], ofx.partial_shape[3]))
one_mat = Concat([one_mat * i for i in dy], axis = 0)
one_mat = Concat([one_mat] * ker_shape, axis = 0)
ofy += one_mat.dimshuffle('x', 0, 1, 2)
ofx = Max(Min(ofx, arg_fea.partial_shape[2] - 1), 0)
ofy = Max(Min(ofy, arg_fea.partial_shape[3] - 1), 0)
def DeformReshape(inp, ker_shape):
inp = inp.reshape(inp.shape[0], ker_shape, ker_shape, inp.shape[2], inp.partial_shape[3])
inp = inp.dimshuffle(0, 3, 1, 4, 2)
inp = inp.reshape(inp.shape[0], inp.shape[1] * inp.shape[2], inp.shape[3] * inp.shape[4])
return inp
ofx = DeformReshape(ofx, ker_shape)
ofy = DeformReshape(ofy, ker_shape)
bilx = DeformReshape(bilx, ker_shape)
bily = DeformReshape(bily, ker_shape)
of = ofx * arg_fea.partial_shape[2] + ofy
arg_fea = arg_fea.reshape(arg_fea.shape[0], arg_fea.shape[1], -1)
of = of.reshape(ofx.shape[0], -1)
of = of.dimshuffle(0, 'x', 1)
#of = Concat([of] * arg_fea.partial_shape[1], axis = 1)
of = of.broadcast((of.shape[0], arg_fea.shape[1], of.shape[2]))
arx = Linspace(0, arg_fea.shape[0], arg_fea.shape[0], endpoint = False)
arx = arx.add_axis(1).add_axis(2).broadcast(of.shape)
ary = Linspace(0, arg_fea.shape[1], arg_fea.shape[1], endpoint = False)
ary = ary.add_axis(0).add_axis(2).broadcast(of.shape)
of = of.add_axis(3)
arx = arx.add_axis(3)
ary = ary.add_axis(3)
idxmap = Astype(Concat([arx, ary, of], axis = 3), np.int32)
"""
sample = []
for i in range(arg_fea.partial_shape[0]):
for j in range(arg_fea.partial_shape[1]):
sample.append(arg_fea[i][j].ai[of[i][j]].dimshuffle('x', 0))
sample = Concat(sample, axis = 0)
"""
sample = IndexingRemap(arg_fea, idxmap).reshape(inp.shape[0], inp.shape[1], bilx.shape[1], -1)
bilx = bilx.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
bily = bily.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
sample *= bilx * bily
outputs.append(sample)
output = outputs[0]
for i in outputs[1:]:
output += i
return Pooling2D(name, output, window = 2, mode = "AVERAGE")
def dfpooling(name, inp, window=2, padding=0, dx=[0, 1], dy=[0, 1]):
#inp = ConstProvider([[[[1, 2], [3, 4]]]], dtype = np.float32)
ker_shape = window
stride = window
gamma = 0.1
offsetx = gamma * inp.partial_shape[2] * Conv2D(name + "offsetx",
inp,
kernel_shape=ker_shape,
stride=stride,
padding=padding,
output_nr_channel=ker_shape
**2,
W=C(0),
nonlinearity=Identity())
offsety = gamma * inp.partial_shape[3] * Conv2D(name + "offsety",
inp,
kernel_shape=ker_shape,
stride=stride,
padding=padding,
output_nr_channel=ker_shape
**2,
W=C(0),
nonlinearity=Identity())
outputs = []
for sx in range(2):
for sy in range(2):
if sx == 0:
ofx = Floor(offsetx)
bilx = offsetx - ofx + Equal(Floor(offsetx), Ceil(offsetx))
else:
ofx = Ceil(offsetx)
bilx = ofx - offsetx
if sy == 0:
ofy = Floor(offsety)
bily = offsety - ofy + Equal(Floor(offsety), Ceil(offsety))
else:
ofy = Ceil(offsety)
bily = ofy - offsety
"""
No padding
padding1 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], 1, inp.partial_shape[3])))
padding2 = ConstProvider(np.zeros((inp.partial_shape[0], inp.partial_shape[1], inp.partial_shape[2] + 2, 1)))
arg_fea = Concat([padding1, inp, padding1], axis = 2)
arg_fea = Concat([padding2, arg_fea, padding2], axis = 3)
"""
arg_fea = inp
#one_mat = ConstProvider(np.ones((inp.partial_shape[2], inp.partial_shape[3])), dtype = np.int32)
one_mat = ConstProvider(1, dtype=np.int32).add_axis(0).broadcast(
(ofx.partial_shape[2], ofx.partial_shape[3]))
affx = (Cumsum(one_mat, axis=0) - 1) * stride
affy = (Cumsum(one_mat, axis=1) - 1) * stride
ofx = ofx + affx.dimshuffle('x', 'x', 0, 1)
ofy = ofy + affy.dimshuffle('x', 'x', 0, 1)
one_mat = ConstProvider(
np.ones(
(ker_shape, ofx.partial_shape[2], ofx.partial_shape[3])))
#ofx[:, :ker_shape, :, :] -= 1
#ofx[:, ker_shape*2:, :, :] += 1
ofx += Concat([one_mat * i for i in dx],
axis=0).dimshuffle('x', 0, 1, 2)
#ofy[:, ::3, :, :] -= 1
#ofy[:, 2::3, :, :] += 1
one_mat = ones((1, ofx.partial_shape[2], ofx.partial_shape[3]))
one_mat = Concat([one_mat * i for i in dy], axis=0)
one_mat = Concat([one_mat] * ker_shape, axis=0)
ofy += one_mat.dimshuffle('x', 0, 1, 2)
ofx = Max(Min(ofx, arg_fea.partial_shape[2] - 1), 0)
ofy = Max(Min(ofy, arg_fea.partial_shape[3] - 1), 0)
def DeformReshape(inp, ker_shape):
inp = inp.reshape(inp.partial_shape[0], ker_shape, ker_shape,
inp.partial_shape[2], inp.partial_shape[3])
inp = inp.dimshuffle(0, 3, 1, 4, 2)
inp = inp.reshape(inp.partial_shape[0],
inp.partial_shape[1] * inp.partial_shape[2],
inp.partial_shape[3] * inp.partial_shape[4])
return inp
ofx = DeformReshape(ofx, ker_shape)
ofy = DeformReshape(ofy, ker_shape)
bilx = DeformReshape(bilx, ker_shape)
bily = DeformReshape(bily, ker_shape)
of = ofx * arg_fea.partial_shape[2] + ofy
arg_fea = arg_fea.reshape(arg_fea.partial_shape[0],
arg_fea.partial_shape[1], -1)
of = of.reshape(ofx.partial_shape[0], -1)
of = of.dimshuffle(0, 'x', 1)
#of = Concat([of] * arg_fea.partial_shape[1], axis = 1)
of = of.broadcast((of.partial_shape[0], arg_fea.partial_shape[1],
of.partial_shape[2]))
arx = Linspace(0,
arg_fea.partial_shape[0],
arg_fea.partial_shape[0],
endpoint=False)
arx = arx.add_axis(1).add_axis(2).broadcast(of.shape)
ary = Linspace(0,
arg_fea.partial_shape[1],
arg_fea.partial_shape[1],
endpoint=False)
ary = ary.add_axis(0).add_axis(2).broadcast(of.shape)
of = of.add_axis(3)
arx = arx.add_axis(3)
ary = ary.add_axis(3)
idxmap = Astype(Concat([arx, ary, of], axis=3), np.int32)
"""
sample = []
for i in range(arg_fea.partial_shape[0]):
for j in range(arg_fea.partial_shape[1]):
sample.append(arg_fea[i][j].ai[of[i][j]].dimshuffle('x', 0))
sample = Concat(sample, axis = 0)
"""
sample = IndexingRemap(arg_fea,
idxmap).reshape(inp.partial_shape[0],
inp.partial_shape[1],
bilx.partial_shape[1], -1)
bilx = bilx.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
bily = bily.dimshuffle(0, 'x', 1, 2).broadcast(sample.shape)
sample *= bilx * bily
outputs.append(sample)
output = outputs[0]
for i in outputs[1:]:
output += i
return Pooling2D(name, output, window=2, mode="AVERAGE")