def test_preprocess():
module = Main()
data = F.ones((1, 14, 8, 8), dtype=np.uint8)
traced_module = trace_module(module, data)
obj = pickle.dumps(traced_module)
traced_module = pickle.loads(obj)
module = Net(traced_module)
module.eval()
idx = F.zeros((1, ), dtype=np.int32)
roi = F.ones((1, 2, 2), dtype=np.float32)
y = module(data, idx, roi)
traced_module = trace_module(module, data, idx, roi)
np.testing.assert_array_equal(traced_module(data, idx, roi), y)
func = trace(traced_module, capture_as_const=True)
np.testing.assert_array_equal(func(data, idx, roi), y)
model = io.BytesIO()
func.dump(model, arg_names=("data", "idx", "roi"))
model.seek(0)
infer_cg = cgtools.GraphInference(model)
np.testing.assert_allclose(
list(
infer_cg.run(inp_dict={
"data": data.numpy(),
"idx": idx.numpy(),
"roi": roi.numpy()
}).values())[0],
y,
atol=1e-6,
)
def test_regression_1762():
x = F.ones((10, 10, 3, 3))
conv = M.Conv2d(10, 10, kernel_size=3, padding=1)
t_shape = (1, 10, 1, 1)
weight = mge.Parameter(np.ones(t_shape, dtype=np.float32))
bias = mge.Parameter(np.zeros(t_shape, dtype=np.float32))
gm = GradManager()
gm.attach(list(conv.parameters()) + [weight, bias])
with gm:
out1 = conv(x)
out2 = F.batch_norm(
out1,
None,
None,
weight,
bias,
training=True,
)
# Weird error only occur when this action is placed after BN
# Op type is not relevant
loss = out1 + 1
gm.backward(loss)
def __init__(self):
super().__init__()
self.A = F.zeros((1, ))
self.I = F.ones((1, ))
self.bb_out = mge.tensor(
np.array([[[0, 0], [160, 0], [160, 48], [0, 48]]],
dtype="float32"))
def test_broadcast():
input1_shape = (20, 30)
output1_shape = (30, 20, 30)
data1 = np.random.random(input1_shape).astype(np.float32)
input2_shape = (10, 1)
output2_shape = (20, 10, 20)
data2 = np.random.random(input2_shape).astype(np.float32)
def compare_fn(x, y):
assert x.shape[0] == y
cases = [
{
"input": [data1, output1_shape],
"output": output1_shape
},
{
"input": [data2, output2_shape],
"output": output2_shape
},
]
opr_test(cases, F.broadcast_to, compare_fn=compare_fn)
x = F.ones((2, 1, 3))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (2, 3, 4))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (4, 1, 3))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (1, 3))
def run_syncbn(trace_mode):
x = F.ones([2, 16, 4, 4], dtype="float32")
net = Sequential(
Conv2d(16, 16, 1), SyncBatchNorm(16), Conv2d(16, 16, 1), SyncBatchNorm(16),
)
gm = ad.GradManager().attach(
net.parameters(), callbacks=dist.make_allreduce_cb("MEAN")
)
opt = optimizer.SGD(net.parameters(), 1e-3)
def train_func(x):
with gm:
y = net(x)
loss = y.mean()
gm.backward(loss)
opt.step().clear_grad()
return loss
if trace_mode is not None:
train_func = trace(train_func, symbolic=trace_mode)
for _ in range(3):
loss = train_func(x)
loss.numpy()
def test_dot():
x = np.random.rand(2, 2).astype("float32")
x = mge.Tensor(x)
u = F.ones((2,))
v = F.ones((2,))
with Grad() as grad:
grad.wrt(x, callback=save_to(x))
def f(x):
return F.dot(u, F.matmul(x, v))
y = f(x)
grad(y, F.ones_like(y))
np.testing.assert_equal(np.ones((2, 2), dtype=np.float32), x.grad.numpy())
def run_frozen_bn(BNModule, is_training, use_trace, use_symbolic):
nchannel = 3
m = BNModule(nchannel, freeze=True)
if is_training:
m.train()
else:
m.eval()
var = 4.0
bias = 1.0
shape = (1, nchannel, 1, 1)
m.running_var[...] = var * F.ones(shape)
m.running_mean[...] = bias * F.ones(shape)
saved_var = m.running_var.numpy()
saved_mean = m.running_mean.numpy()
saved_wt = m.weight.numpy()
saved_bias = m.bias.numpy()
gm = ad.GradManager().attach(m.parameters())
optim = optimizer.SGD(m.parameters(), lr=1.0)
optim.clear_grad()
data = np.random.random((6, nchannel, 2, 2)).astype("float32")
def train_fn(d):
for _ in range(3):
with gm:
loss = m(d).mean()
gm.backward(loss)
optim.step()
return loss
if use_trace:
train_fn = trace(train_fn, symbolic=use_symbolic)
for _ in range(3):
loss = train_fn(megengine.tensor(data))
if not is_training:
np.testing.assert_equal(m.running_var.numpy(), saved_var)
np.testing.assert_equal(m.running_mean.numpy(), saved_mean)
np.testing.assert_almost_equal(
loss.numpy(), ((data - bias) / np.sqrt(var)).mean(), 5
)
np.testing.assert_equal(m.weight.numpy(), saved_wt)
np.testing.assert_equal(m.bias.numpy(), saved_bias)
def fpn_anchor_target_opr_core_impl(gt_boxes,
im_info,
anchors,
allow_low_quality_matches=True):
ignore_label = config.ignore_label
# get the gt boxes
gtboxes = gt_boxes[:im_info[5].astype(np.int32)]
ignore_mask = F.equal(gtboxes[:, 4], config.ignore_label)
# find the valid gtboxes
_, index = F.cond_take(1 - ignore_mask > 0, ignore_mask)
valid_gt_boxes = gtboxes[index.astype(np.int32)]
# compute the iou matrix
overlaps = box_overlap_opr(anchors, valid_gt_boxes[:, :4])
# match the dtboxes
a_shp0 = anchors.shape[0]
argmax_overlaps = F.argmax(overlaps, axis=1)
max_overlaps = F.nn.indexing_one_hot(overlaps,
argmax_overlaps.astype(np.int32), 1)
labels = F.ones(a_shp0).astype(np.int32) * ignore_label
# set negative ones
labels = labels * (max_overlaps >= config.rpn_negative_overlap).astype(
np.float32)
# set positive ones
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
const_one = mge.tensor(1.0)
if allow_low_quality_matches:
# match the max gt
gt_max_overlaps = F.max(overlaps, axis=0)
gt_argmax_overlaps = F.argmax(overlaps, axis=0)
gt_argmax_overlaps = gt_argmax_overlaps.astype(np.int32)
max_overlaps[gt_argmax_overlaps] = 1.
m = gt_max_overlaps.shape[0]
argmax_overlaps[gt_argmax_overlaps] = F.linspace(0, m - 1,
m).astype(np.int32)
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
labels[fg_mask] = 1
# compute the bbox targets
bbox_targets = bbox_transform_opr(anchors,
valid_gt_boxes[argmax_overlaps, :4])
if config.rpn_bbox_normalize_targets:
std_opr = mge.tensor(config.bbox_normalize_stds[None, :]).to(
anchors.device)
mean_opr = mge.tensor(config.bbox_normalize_means[None, :]).to(
anchors.device)
minus_opr = mean_opr / std_opr
bbox_targets = bbox_targets / std_opr - minus_opr
return labels, bbox_targets
def forward(self, pred_cls_list, rpn_num_prob_list, pred_reg_list,
anchors_list, rpn_iou_list, boxes, im_info):
all_anchors_list = [
F.concat([a, i * F.ones([a.shape[0], 1]).to(a.device)], axis=1)
for i, a in enumerate(anchors_list)
]
all_anchors_final = F.concat(all_anchors_list, axis=0)
rpn_bbox_offset_final = F.concat(pred_reg_list, axis=1)
rpn_cls_prob_final = F.concat(pred_cls_list, axis=1)
rpn_iou_prob_final = F.concat(rpn_iou_list, axis=1)
rpn_num_per_points_final = F.concat(rpn_num_prob_list, axis=1)
rpn_labels, rpn_target_boxes = rpn_anchor_target_opr(
boxes, im_info, all_anchors_final)
ious_target = self.anchor_iou_target_opr(boxes, im_info,
all_anchors_final,
rpn_bbox_offset_final)
n = rpn_labels.shape[0]
target_boxes = rpn_target_boxes.reshape(n, -1, 4)
rpn_cls_prob_final = rpn_cls_prob_final.reshape(n, -1, 1)
offsets_final = rpn_bbox_offset_final.reshape(n, -1, 4)
rpn_labels = rpn_labels.transpose(2, 0, 1)
a, b = rpn_labels[0], rpn_labels[1]
ignores = b - F.equal(a, 0).astype(np.float32) * F.equal(b, 0).astype(
np.float32)
labels = F.stack([a, ignores], axis=2).reshape(n, -1)
cls_loss = sigmoid_cross_entropy_retina(rpn_cls_prob_final,
labels,
alpha=config.focal_loss_alpha,
gamma=config.focal_loss_gamma)
rpn_bbox_loss = smooth_l1_loss_retina(offsets_final, target_boxes,
labels)
rpn_labels = labels.reshape(n, -1, 2)
rpn_iou_loss = iou_l1_loss(rpn_iou_prob_final, ious_target, rpn_labels)
# whether one anchor produce one proposal or two.
nlabels = ((labels.reshape(n, -1, 2) > 0).sum(2)).flatten() - 1
c = rpn_num_per_points_final.shape[2]
num_per_anchor = rpn_num_per_points_final.reshape(-1, c)
rpn_num_per_points_final = rpn_num_per_points_final.reshape(-1, c)
nlabels = nlabels.reshape(-1)
rpn_num_loss = softmax_loss(rpn_num_per_points_final, nlabels)
loss_dict = {}
loss_dict['rpn_cls_loss'] = cls_loss
loss_dict['rpn_bbox_loss'] = 2 * rpn_bbox_loss
loss_dict['rpn_iou_loss'] = 2 * rpn_iou_loss
loss_dict['rpn_num_loss'] = rpn_num_loss
return loss_dict
def test_splice_network():
x = F.ones((2, ))
y = F.ones((2, ))
@trace(symbolic=True, capture_as_const=True)
def fun1(a, b):
return (a + b) * 2
@trace(symbolic=True, capture_as_const=True)
def fun2(a):
return a * 2 - 1
model = io.BytesIO()
fun1(x, y)
fun2(x)
fun1.dump(
model,
arg_names=["net1_i0", "net1_i1"],
output_names=["net1_o0"],
optimize_for_inference=False,
)
model.seek(0)
net1 = Net.load(model)
model.seek(0)
fun2.dump(
model,
arg_names=["net2_i0"],
output_names=["net2_o0"],
optimize_for_inference=False,
)
model.seek(0)
net2 = Net.load(model)
net1.add_output(*net2.output_vars)
var = net1.var_filter.name("net1_i0").as_unique()
repl_var = net2.var_filter.name("net2_o0").as_unique()
net1.replace_vars({var: repl_var})
assert "net1_i0" not in [var.name for var in net1.all_vars]
assert "net2_i0" in [var.name for var in net1.all_vars]
model.seek(0)
net1.dump(model, keep_var_name=2, optimize_for_inference=False)
model.seek(0)
net = Net.load(model)
assert "net1_i0" not in [var.name for var in net.all_vars]
assert "net2_i0" in [var.name for var in net.all_vars]
def test_empty_grad_in_backward():
x = mge.Parameter(F.full(100, 0.5))
y = mge.Parameter(F.ones(100))
gm = GradManager()
gm.attach([x, y])
with gm:
z = F.where(x > 0.7, x, y)
loss = z.sum()
gm.backward(loss)
assert np.all(x.grad.numpy() == 0)
assert np.all(y.grad.numpy() == 1)
def forward(self, fpn_fms, rcnn_rois, gt_boxes=None, im_info=None):
if self.training:
loss = {}
for i, _ in enumerate(self.iou_thrs):
loss_dict, prob = self.subnets[i](fpn_fms, rcnn_rois, gt_boxes,
im_info)
rois = prob[:, 1]
rcnn_list = []
for bid in range(config.batch_per_gpu):
mask = F.equal(rois[:, 5], bid)
_, index = F.cond_take(mask > 0, mask)
batch_id = bid * F.ones([mask.sum(), 1])
m = F.concat([batch_id, rois[index, :4]], axis=1)
rcnn_list.append(m)
rcnn_rois = F.concat(rcnn_list, axis=0)
loss.update(loss_dict)
return loss
else:
# boxes_pred = self._forward_test(fpn_fms, rcnn_rois)
for i, _ in enumerate(self.iou_thrs):
prob = self.subnets[i](fpn_fms, rcnn_rois)
rois = prob[:, 1]
rcnn_list = []
for bid in range(1):
mask = F.equal(rois[:, 5], bid)
_, index = F.cond_take(mask > 0, mask)
batch_id = bid * F.ones([mask.sum(), 1])
m = F.concat([batch_id, rois[index, :4]], axis=1)
rcnn_list.append(m)
rcnn_rois = F.concat(rcnn_list, axis=0)
return prob[:, :, :5]
def train_pipeline():
m = ResNet18Pipeline()
x = F.ones([32, 3, 224, 224])
label = F.zeros([
32,
], dtype="int32")
gm = ad.GradManager().attach(m.parameters())
opt = optim.SGD(m.parameters(), 1e-3, 0.9, 1e-4)
for _ in range(2):
m(x)
loss = m.backward(label, gm)
opt.step().clear_grad()
print(loss)
def test_jit_trace():
module = MyModule()
module.eval()
x = F.ones((1, 8, 14, 14))
expect = module(x)
traced_module = trace_module(module, x)
func = trace(traced_module, capture_as_const=True)
np.testing.assert_array_equal(func(x), expect)
model = io.BytesIO()
func.dump(model)
model.seek(0)
infer_cg = cgtools.GraphInference(model)
np.testing.assert_allclose(list(infer_cg.run(x.numpy()).values())[0],
expect,
atol=1e-6)
def create_mask_mge(tensor, paddings):
shape = tensor.shape # N,c, H,W
inner_width = shape[2] - (paddings[0][0] + paddings[0][1])
inner_height = shape[3] - (paddings[1][0] + paddings[1][1])
inner_mge = F.ones([shape[0], shape[1], inner_width,
inner_height]) # .float() # need check cuda?
outer_mge = F.zeros([shape[0], shape[1], shape[2],
shape[3]]) # .float()
outer_mge[:, :, paddings[0][0]:paddings[0][0] + inner_width,
paddings[1][0]:paddings[1][0] + inner_height] = inner_mge
# if tensor.is_cuda:
# inner_torch = inner_torch.cuda()
# mask2d = F.pad(inner_mge, [paddings[0][0], paddings[0][1], paddings[1][0], paddings[1][1]]) # no padding layer
return outer_mge
def worker():
x = F.ones([3, 10], dtype="float32")
m = M.Linear(10, 10)
def func():
with GradManager().attach(m.parameters()) as gm:
if dist.get_rank() == 0:
y = m(x)
else:
y = x
y = F.distributed.broadcast(y)
gm.backward(y)
if trace_mode is not None:
func = trace(symbolic=trace_mode)(func)
func()
def test_broadcast(is_varnode):
if is_varnode:
network = Network()
else:
network = None
input1_shape = (20, 30)
output1_shape = (30, 20, 30)
data1 = np.random.random(input1_shape).astype(np.float32)
input2_shape = (10, 1)
output2_shape = (20, 10, 20)
data2 = np.random.random(input2_shape).astype(np.float32)
input3_shape = (10, 10)
output3_shape = (10, 10)
data3 = np.random.random(input3_shape).astype(np.float32)
def compare_fn(x, y):
assert x._tuple_shape[0] == y
cases = [
{
"input": [data1, output1_shape],
"output": output1_shape
},
{
"input": [data2, output2_shape],
"output": output2_shape
},
{
"input": [data3, output3_shape],
"output": output3_shape
},
]
opr_test(cases, F.broadcast_to, compare_fn=compare_fn, network=network)
x = F.ones((2, 1, 3))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (2, 3, 4))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (4, 1, 3))
with pytest.raises(RuntimeError):
F.broadcast_to(x, (1, 3))
def worker():
m = M.Linear(10, 10)
x = F.ones([3, 10], dtype="float32")
def func():
with GradManager().attach(m.parameters()) as gm:
y = m(x)
y = F.distributed.reduce_sum(y)
if dist.get_rank() == 0:
loss = (2 * y + 1).mean()
gm.backward(loss)
else:
gm.backward()
if trace_mode is not None:
func = trace(symbolic=trace_mode)(func)
func()
def forward(self, pred_cls_list, rpn_num_prob_list, pred_reg_list,
anchors_list, rpn_iou_list, boxes, im_info):
all_anchors_list = [
F.concat([a, i * F.ones([a.shape[0], 1]).to(a.device)], axis=1)
for i, a in enumerate(anchors_list)
]
all_anchors_final = F.concat(all_anchors_list, axis=0)
rpn_bbox_offset_final = F.concat(pred_reg_list, axis=1)
rpn_cls_prob_final = F.concat(pred_cls_list, axis=1)
rpn_iou_prob_final = F.concat(rpn_iou_list, axis=1)
rpn_num_per_points_final = F.concat(rpn_num_prob_list, axis=1)
rpn_labels, rpn_target_boxes = rpn_anchor_target_opr(
boxes, im_info, all_anchors_final)
ious_target = self.anchor_iou_target_opr(boxes, im_info,
all_anchors_final,
rpn_bbox_offset_final)
n = rpn_labels.shape[0]
target_boxes = rpn_target_boxes.reshape(n, -1, 2,
4).transpose(2, 0, 1, 3)
rpn_cls_prob_final = rpn_cls_prob_final
offsets_final = rpn_bbox_offset_final
target_boxes = target_boxes[0]
rpn_labels = rpn_labels.transpose(2, 0, 1)
labels = rpn_labels[0]
cls_loss = sigmoid_cross_entropy_retina(rpn_cls_prob_final,
labels,
alpha=config.focal_loss_alpha,
gamma=config.focal_loss_gamma)
rpn_bbox_loss = smooth_l1_loss_retina(offsets_final, target_boxes,
labels)
rpn_labels = F.expand_dims(labels, axis=2)
rpn_iou_loss = iou_l1_loss(rpn_iou_prob_final, ious_target, rpn_labels)
loss_dict = {}
loss_dict['rpn_cls_loss'] = cls_loss
loss_dict['rpn_bbox_loss'] = 2 * rpn_bbox_loss
loss_dict['rpn_iou_loss'] = 2 * rpn_iou_loss
return loss_dict
def train():
m = ResNet18MP()
x = F.ones([32, 3, 224, 224])
label = F.zeros([
32,
], dtype="int32")
gm = ad.GradManager().attach(m.parameters())
opt = optim.SGD(m.parameters(), 1e-3, 0.9, 1e-4)
for _ in range(2):
with gm:
y = m(x)
if dist.get_rank() == 3:
loss = F.nn.cross_entropy(y, label)
else:
loss = None
gm.backward(loss)
opt.step().clear_grad()
print(loss)
def get_padded_tensor(array: Tensor,
multiple_number: int = 32,
pad_value: float = 0) -> Tensor:
""" pad the nd-array to multiple stride of th e
Args:
array (Tensor):
the tensor with the shape of [batch, channel, height, width]
multiple_number (int):
make the height and width can be divided by multiple_number
pad_value (int): the value to be padded
Returns:
padded_array (Tensor)
"""
batch, chl, t_height, t_width = array.shape
padded_height = ((t_height + multiple_number - 1) // multiple_number *
multiple_number)
padded_width = (t_width + multiple_number -
1) // multiple_number * multiple_number
padded_array = F.ones([batch, chl, t_height, t_width],
dtype=np.float32) * pad_value
# padded_array = (
# F.ones(
# F.concat([batch, chl, padded_height, padded_width], axis=0),
# dtype=np.float32,
# )
# * pad_value
# )
ndim = array.ndim
if ndim == 4:
padded_array[:, :, :t_height, :t_width] = array
# padded_array = padded_array.set_subtensor(array)[:, :, :t_height, :t_width]
elif ndim == 3:
# padded_array = padded_array.set_subtensor(array)[:, :t_height, :t_width]
padded_array[:, :, :t_height, :t_width] = array
else:
raise Exception("Not supported tensor dim: %d" % ndim)
return padded_array
def sigmoid_cross_entropy_retina(pred,
label,
ignore_label=-1,
background=0,
alpha=0.5,
gamma=0):
device = pred.device
mask = 1 - F.equal(label, ignore_label).astype(np.float32)
vlabel = label * mask
n, m, c = pred.shape
zero_mat = F.zeros([n, m, c + 1]).to(device)
index = F.expand_dims(vlabel, 2).astype(np.int32)
one_hot = F.scatter(zero_mat, 2, index, F.ones([n, m, 1]))
onehot = one_hot[:, :, 1:]
pos_part = F.pow(1 - pred, gamma) * onehot * F.log(pred)
neg_part = F.pow(pred, gamma) * (1 - onehot) * F.log(1 - pred)
loss = -(alpha * pos_part + (1 - alpha) * neg_part).sum(axis=2) * mask
positive_mask = (label > 0)
return loss.sum() / F.maximum(positive_mask.sum(), 1)
def test_ones(val):
shp = tensor(val)
np_shp = np.array(val)
np.testing.assert_equal(F.ones(shp), np.ones(np_shp))
def __init__(self):
super().__init__()
self.I = F.ones((1, ))
self.M = F.zeros((1, ))
def _photo_loss_census(img1,
img2_warp,
occu_mask1=None,
max_distance=3,
loss_type='L1',
if_mask=False):
patch_size = 2 * max_distance + 1
def _ternary_transform_mge(image):
n, c, h, w = image.shape
if c == 3:
R, G, B = F.split(image, 3, 1)
intensities = (0.2989 * R + 0.5870 * G + 0.1140 * B
) # * 255 # convert to gray
elif c == 1:
intensities = image
else:
raise ValueError('image channel should be 3 or 1: %s' % c)
# intensities = tf.image.rgb_to_grayscale(image) * 255
out_channels = patch_size * patch_size
w = np.eye(out_channels).reshape(
(patch_size, patch_size, 1, out_channels)) # h,w,1,out_c
w_ = np.transpose(w, (3, 2, 0, 1)) # 1,out_c,h,w
# weight = torch.from_numpy(w_).float()
weight = mge.tensor(w_.astype(np.float32)) # need check cuda?
# if image.is_cuda:
# weight = weight.cuda()
# patches_torch = torch.conv2d(input=out_channels, weight=weight, bias=None, stride=[1, 1], padding=[max_distance, max_distance])
patches_mge = F.nn.conv2d(inp=intensities,
weight=weight,
bias=None,
stride=[1, 1],
padding=[max_distance, max_distance])
transf_mge = patches_mge - intensities
transf_norm_mge = transf_mge / F.sqrt(0.81 + transf_mge**2)
return transf_norm_mge
def _hamming_distance_mge(t1, t2):
dist = (t1 - t2)**2
dist = F.sum(dist / (0.1 + dist), axis=1, keepdims=True)
return dist
def create_mask_mge(tensor, paddings):
shape = tensor.shape # N,c, H,W
inner_width = shape[2] - (paddings[0][0] + paddings[0][1])
inner_height = shape[3] - (paddings[1][0] + paddings[1][1])
inner_mge = F.ones([shape[0], shape[1], inner_width,
inner_height]) # .float() # need check cuda?
outer_mge = F.zeros([shape[0], shape[1], shape[2],
shape[3]]) # .float()
outer_mge[:, :, paddings[0][0]:paddings[0][0] + inner_width,
paddings[1][0]:paddings[1][0] + inner_height] = inner_mge
# if tensor.is_cuda:
# inner_torch = inner_torch.cuda()
# mask2d = F.pad(inner_mge, [paddings[0][0], paddings[0][1], paddings[1][0], paddings[1][1]]) # no padding layer
return outer_mge
# ==== photo loss functions ====
def _L1(diff, occ_mask=None, if_mask_=False):
loss_diff = F.abs(diff)
if not if_mask_:
photo_loss = F.mean(loss_diff)
else:
photo_loss = F.sum(loss_diff * occ_mask) / (F.sum(occ_mask) + 1e-6)
return photo_loss
def _charbonnier(diff, occ_mask=None, if_mask_=False):
loss_diff = F.pow((diff**2 + 1e-6), 0.4)
if not if_mask_:
photo_loss = F.mean(loss_diff)
else:
photo_loss = F.sum(loss_diff * occ_mask) / (F.sum(occ_mask) + 1e-6)
return photo_loss
def _abs_robust(diff, occ_mask=None, if_mask_=False):
loss_diff = F.pow((F.abs(diff) + 0.01), 0.4)
if not if_mask_:
photo_loss = F.mean(loss_diff)
else:
photo_loss = F.sum(loss_diff * occ_mask) / (F.sum(occ_mask) + 1e-6)
return photo_loss
img1 = _ternary_transform_mge(img1)
img2_warp = _ternary_transform_mge(img2_warp)
dist = _hamming_distance_mge(img1, img2_warp)
if occu_mask1 is None:
im_shape = img1.shape
occu_mask1 = F.ones([im_shape[0], 1, im_shape[2],
im_shape[3]]) # .float()
transform_mask = create_mask_mge(
occu_mask1,
[[max_distance, max_distance], [max_distance, max_distance]])
occ_mask = occu_mask1 * transform_mask
# ===== compute photo loss =====
if loss_type == 'L1':
census_loss = _L1(dist, occ_mask, if_mask_=if_mask)
elif loss_type == 'abs_robust':
census_loss = _abs_robust(dist, occ_mask, if_mask_=if_mask)
elif loss_type == 'charbonnier':
census_loss = _charbonnier(dist, occ_mask, if_mask_=if_mask)
else:
raise ValueError('wrong photo loss type in census loss: %s' %
loss_type)
return census_loss
def get_ground_truth(self, anchors_list, batched_gt_boxes,
batched_num_gts):
labels_list = []
offsets_list = []
ctrness_list = []
all_level_anchors = F.concat(anchors_list, axis=0)
for bid in range(batched_gt_boxes.shape[0]):
gt_boxes = batched_gt_boxes[bid, :batched_num_gts[bid]]
ious = []
candidate_idxs = []
base = 0
for stride, anchors_i in zip(self.cfg.stride, anchors_list):
ious.append(
layers.get_iou(
gt_boxes[:, :4],
F.concat([
anchors_i - stride * self.cfg.anchor_scale / 2,
anchors_i + stride * self.cfg.anchor_scale / 2,
],
axis=1)))
gt_centers = (gt_boxes[:, :2] + gt_boxes[:, 2:4]) / 2
distances = F.sqrt(
F.sum((F.expand_dims(gt_centers, axis=1) - anchors_i)**2,
axis=2))
_, topk_idxs = F.topk(distances, self.cfg.anchor_topk)
candidate_idxs.append(base + topk_idxs)
base += anchors_i.shape[0]
ious = F.concat(ious, axis=1)
candidate_idxs = F.concat(candidate_idxs, axis=1)
candidate_ious = F.gather(ious, 1, candidate_idxs)
ious_thr = (F.mean(candidate_ious, axis=1, keepdims=True) +
F.std(candidate_ious, axis=1, keepdims=True))
is_foreground = F.scatter(
F.zeros(ious.shape), 1, candidate_idxs,
F.ones(candidate_idxs.shape)).astype(bool) & (ious >= ious_thr)
is_in_boxes = F.min(self.point_coder.encode(
all_level_anchors, F.expand_dims(gt_boxes[:, :4], axis=1)),
axis=2) > 0
ious[~is_foreground] = -1
ious[~is_in_boxes] = -1
match_indices = F.argmax(ious, axis=0)
gt_boxes_matched = gt_boxes[match_indices]
anchor_max_iou = F.indexing_one_hot(ious, match_indices, axis=0)
labels = gt_boxes_matched[:, 4].astype(np.int32)
labels[anchor_max_iou == -1] = 0
offsets = self.point_coder.encode(all_level_anchors,
gt_boxes_matched[:, :4])
left_right = offsets[:, [0, 2]]
top_bottom = offsets[:, [1, 3]]
ctrness = F.sqrt(
F.clip(F.min(left_right, axis=1) / F.max(left_right, axis=1),
lower=0) *
F.clip(F.min(top_bottom, axis=1) / F.max(top_bottom, axis=1),
lower=0))
labels_list.append(labels)
offsets_list.append(offsets)
ctrness_list.append(ctrness)
return (
F.stack(labels_list, axis=0).detach(),
F.stack(offsets_list, axis=0).detach(),
F.stack(ctrness_list, axis=0).detach(),
)
def fpn_roi_target(rpn_rois,
im_info,
gt_boxes,
fg_threshold=config.fg_threshold,
top_k=1):
return_rois, return_labels = [], []
return_bbox_targets = []
# get per image proposals and gt_boxes
batch_per_gpu = im_info.shape[0]
sampling = True
# is_sample = True if top_k < 2 else False
for bid in range(batch_per_gpu):
gt_boxes_perimg = gt_boxes[bid, :im_info[bid, 5].astype(np.int32), :]
dummy_gt = F.ones([1, gt_boxes_perimg.shape[1]])
batch_inds = F.ones((gt_boxes_perimg.shape[0], 1)) * bid
#if config.proposal_append_gt:
gt_rois = F.concat([batch_inds, gt_boxes_perimg[:, :4]], axis=1)
batch_rois_mask = F.equal(rpn_rois[:, 0], bid) > 0
_, batch_rois_index = F.cond_take(batch_rois_mask, batch_rois_mask)
# batch_roi_mask = rpn_rois[:, 0] == bid
# batch_roi_inds = mask_to_inds(batch_roi_mask)
all_rois= F.concat([rpn_rois[batch_rois_index], gt_rois], axis=0) if sampling \
else rpn_rois[batch_rois_index]
# all_rois = F.concat([rpn_rois.ai[batch_roi_inds], gt_rois], axis=0)
gt_boxes_perimg = F.concat([gt_boxes_perimg, dummy_gt], axis=0)
overlaps_normal, overlaps_ignore = box_overlap_ignore_opr(
all_rois[:, 1:5], gt_boxes_perimg)
# overlaps_normal, overlaps_normal_indices = F.argsort(overlaps_normal, descending=True)
# overlaps_ignore, overlaps_ignore_indices = F.argsort(overlaps_ignore, descending=True)
overlaps_normal_indices = F.argsort(overlaps_normal, descending=True)
overlaps_normal = F.gather(overlaps_normal, 1, overlaps_normal_indices)
# overlaps_normal = F.nn.indexing_one_hot(overlaps_normal, overlaps_normal_indices, 1)
overlaps_ignore_indices = F.argsort(overlaps_ignore, descending=True)
overlaps_ignore = F.gather(overlaps_ignore, 1, overlaps_ignore_indices)
# overlaps_ignore = F.nn.indexing_one_hot(overlaps_ignore, overlaps_ignore_indices, 1)
# gt max and indices, ignore max and indices
max_overlaps_normal = overlaps_normal[:, :top_k].flatten()
gt_assignment_normal = overlaps_normal_indices[:, :top_k].flatten()
max_overlaps_ignore = overlaps_ignore[:, :top_k].flatten()
gt_assignment_ignore = overlaps_ignore_indices[:, :top_k].flatten()
# cons masks
ignore_assign_mask = (max_overlaps_normal < fg_threshold).astype(
np.float32) * (max_overlaps_ignore > max_overlaps_normal).astype(
np.float32)
max_overlaps = max_overlaps_normal * (1 - ignore_assign_mask).astype(np.float32) + \
max_overlaps_ignore * ignore_assign_mask
gt_assignment = gt_assignment_normal * (1- ignore_assign_mask) + \
gt_assignment_ignore * ignore_assign_mask
gt_assignment = gt_assignment.astype(np.int32)
labels = gt_boxes_perimg[gt_assignment, 4]
fg_mask = (max_overlaps >= fg_threshold).astype(
np.float32) * (1 - F.equal(labels, config.ignore_label))
bg_mask = (max_overlaps < config.bg_threshold_high).astype(
np.float32) * (max_overlaps >= config.bg_threshold_low).astype(
np.float32)
fg_mask = fg_mask.reshape(-1, top_k)
bg_mask = bg_mask.reshape(-1, top_k)
pos_max = config.num_rois * config.fg_ratio
fg_inds_mask = _bernoulli_sample_masks(
fg_mask[:,
0], pos_max, 1) if sampling else F.equal(fg_mask[:, 0], 0)
neg_max = config.num_rois - fg_inds_mask.sum()
bg_inds_mask = _bernoulli_sample_masks(
bg_mask[:,
0], neg_max, 1) if sampling else F.equal(bg_mask[:, 0], 0)
labels = labels * fg_mask.reshape(-1)
keep_mask = fg_inds_mask + bg_inds_mask
keep_mask = keep_mask + F.equal(keep_mask.sum(), 0)
# keep_inds = mask_to_inds(keep_mask)
_, keep_inds = F.cond_take(keep_mask > 0, keep_mask)
#keep_inds = keep_inds[:F.minimum(config.num_rois, keep_inds.shapeof()[0])]
# labels
labels = labels.reshape(-1, top_k)[keep_inds]
gt_assignment = gt_assignment.reshape(
-1, top_k)[keep_inds].reshape(-1).astype(np.int32)
target_boxes = gt_boxes_perimg[gt_assignment, :4]
# rois = all_rois.ai[keep_inds]
rois = all_rois[keep_inds]
# target_shape = (rois.shapeof()[0], top_k, rois.shapeof()[-1])
n, c = rois.shape[0], rois.shape[1]
target_rois = F.broadcast_to(F.expand_dims(rois, 1),
(n, top_k, c)).reshape(-1, c)
# target_rois = F.add_axis(rois, 1).broadcast(target_shape).reshape(-1, rois.shapeof()[-1])
bbox_targets = bbox_transform_opr(target_rois[:, 1:5],
target_boxes[:, :4])
if config.rcnn_bbox_normalize_targets:
std_opr = mge.tensor(config.bbox_normalize_stds[None, :]).to(
rois.device)
mean_opr = mge.tensor(config.bbox_normalize_means[None, :]).to(
rois.device)
minus_opr = mean_opr / std_opr
bbox_targets = bbox_targets / std_opr - minus_opr
bbox_targets = bbox_targets.reshape(-1, top_k * 4)
return_rois.append(rois)
return_labels.append(labels)
return_bbox_targets.append(bbox_targets)
if config.batch_per_gpu == 1:
rois, labels, bbox_targets = rois.detach(), labels.detach(
), bbox_targets.detach()
return rois, labels, bbox_targets
# return F.zero_grad(rois), F.zero_grad(labels), F.zero_grad(bbox_targets)
else:
return_rois = F.concat(return_rois, axis=0)
return_labels = F.concat(return_labels, axis=0)
return_bbox_targets = F.concat(return_bbox_targets, axis=0)
return_rois = return_rois.detach()
return_labels = return_labels.detach()
return_bbox_targets = return_bbox_targets.detach()
return return_rois, return_labels, return_bbox_targets
def test_assert_equal():
shape = (2, 3, 4, 5)
x = F.ones(shape, dtype=np.float32)
y = F.zeros(shape, dtype=np.float32) + 1.00001
z = F.utils._assert_equal(x, y)
def test_assert_not_equal():
shape = (2, 3, 4, 5)
x = F.ones(shape, dtype=np.float32)
y = F.zeros(shape, dtype=np.float32) + 1.1
with pytest.raises(RuntimeError):
z = F.utils._assert_equal(x, y)
def find_top_rpn_proposals(is_train, rpn_bbox_offsets_list, rpn_cls_prob_list,
all_anchors_list, im_info):
prev_nms_top_n = config.train_prev_nms_top_n \
if is_train else config.test_prev_nms_top_n
post_nms_top_n = config.train_post_nms_top_n \
if is_train else config.test_post_nms_top_n
batch_per_gpu = config.batch_per_gpu if is_train else 1
nms_threshold = config.rpn_nms_threshold
box_min_size = config.rpn_min_box_size
bbox_normalize_targets = config.rpn_bbox_normalize_targets
bbox_normalize_means = config.bbox_normalize_means
bbox_normalize_stds = config.bbox_normalize_stds
list_size = len(rpn_bbox_offsets_list)
return_rois, return_probs = [], []
batch_per_gpu = rpn_cls_prob_list[0].shape[0]
for bid in range(batch_per_gpu):
batch_proposals_list = []
batch_probs_list = []
for l in range(list_size):
# get proposals and probs
offsets = rpn_bbox_offsets_list[l][bid] \
.transpose(1, 2, 0).reshape(-1, 4)
if bbox_normalize_targets:
std_opr = tensor(config.bbox_normalize_stds[None, :])
mean_opr = tensor(config.bbox_normalize_means[None, :])
pred_offsets = pred_offsets * std_opr
pred_offsets = pred_offsets + mean_opr
all_anchors = all_anchors_list[l]
proposals = bbox_transform_inv_opr(all_anchors, offsets)
if config.anchor_within_border:
proposals = clip_boxes_opr(proposals, im_info[bid, :])
probs = rpn_cls_prob_list[l][bid] \
.transpose(1,2,0).reshape(-1, 2)
probs = F.softmax(probs)[:, 1]
# gather the proposals and probs
batch_proposals_list.append(proposals)
batch_probs_list.append(probs)
batch_proposals = F.concat(batch_proposals_list, axis=0)
batch_probs = F.concat(batch_probs_list, axis=0)
# filter the boxes with small size.
wh = batch_proposals[:, 2:4] - batch_proposals[:, :2] + 1
thresh = box_min_size * im_info[bid, 2]
keep_mask = F.prod((wh >= thresh), axis=1)
keep_mask = keep_mask + F.equal(keep_mask.sum(), 0)
keep_mask, inds = F.cond_take(keep_mask > 0, keep_mask)
inds = inds.astype(np.int32)
# batch_proposals = F.nn.indexing_one_hot(batch_proposals, inds, 0)
# batch_probs = F.nn.indexing_one_hot(batch_probs, inds, 0)
batch_proposals, batch_probs = batch_proposals[inds], batch_probs[inds]
# prev_nms_top_n
num_proposals = F.minimum(prev_nms_top_n, batch_proposals.shape[0])
idx = F.argsort(batch_probs, descending=True)
topk_idx = idx[:num_proposals].reshape(-1)
batch_proposals = batch_proposals[topk_idx].detach()
batch_probs = batch_probs[topk_idx].detach()
# For each image, run a total-level NMS, and choose topk results.
keep_inds = nms(batch_proposals,
batch_probs,
nms_threshold,
max_output=2000)
# num = F.minimum(post_nms_top_n, keep_inds.shape[0])
# keep_inds = keep_inds[:num]
batch_rois, batch_probs = batch_proposals[keep_inds], batch_probs[
keep_inds]
# cons the rois
batch_inds = F.ones((batch_rois.shape[0], 1)) * bid
batch_rois = F.concat([batch_inds, batch_rois[:, :4]], axis=1)
return_rois.append(batch_rois)
return_probs.append(batch_probs)
if batch_per_gpu == 1:
return batch_rois, batch_probs
else:
concated_rois = F.concat(return_rois, axis=0)
concated_probs = F.concat(return_probs, axis=0)
return concated_rois, concated_probs