def pyapply_deltas(self, datas, img_size=None):
'''
'''
h_ct = tf.nn.sigmoid(datas['heatmaps_ct'])
offset = datas['offset']
hw = datas['hw']
B, H, W, C = wmlt.combined_static_and_dynamic_shape(h_ct)
offset = tf.reshape(offset, [B, -1, 2])
hw = tf.reshape(hw, [B, -1, 2])
h_ct = self.pixel_nms(h_ct, threshold=self.score_threshold)
ct_scores, ct_inds, ct_clses, ct_ys, ct_xs = self._topk(h_ct, k=self.k)
C = btf.channel(h_ct)
hw_inds = ct_inds // C
K = self.k
ct_ys = tf.reshape(ct_ys, [B, K])
ct_xs = tf.reshape(ct_xs, [B, K])
offset = wmlt.batch_gather(offset, hw_inds)
offset = tf.reshape(offset, [B, K, 2])
offset_y, offset_x = tf.unstack(offset, axis=-1)
ct_xs = ct_xs + offset_x
ct_ys = ct_ys + offset_y
hw = wmlt.batch_gather(hw, hw_inds)
hw = tf.reshape(hw, [B, K, 2])
h, w = tf.unstack(hw, axis=-1)
ymin, xmin, ymax, xmax = [
ct_ys - h / 2, ct_xs - w / 2, ct_ys + h / 2, ct_xs + w / 2
]
bboxes = tf.stack([ymin, xmin, ymax, xmax], axis=-1)
bboxes = odb.tfabsolutely_boxes_to_relative_boxes(bboxes,
width=W,
height=H)
return bboxes, ct_clses, ct_scores, hw_inds
def merge_bboxes_and_mask_results(self, instances):
nms = functools.partial(tfop.boxes_nms,
threshold=self.test_nms_thresh,
classes_wise=True)
mask = tf.squeeze(instances[RD_MASKS], axis=0)
bmask = tf.expand_dims(mask, axis=-1)
bmask = wnnl.min_pool2d(bmask, kernel_size=2, stride=1, padding="SAME")
bmask = slim.max_pool2d(bmask, kernel_size=2, stride=1, padding="SAME")
bmask = tf.squeeze(bmask, axis=-1)
bmask = tf.cast(bmask * 255, tf.uint8)
#bboxes = tf.squeeze(instances[RD_BOXES],axis=0)
bboxes = tfop.get_bboxes_from_mask(bmask)
#bboxes = tf.Print(bboxes,["xbboxes",tf.reduce_max(xboxes,keepdims=False),
#tf.reduce_max(mask)],summarize=100)
Nr, H, W = btf.combined_static_and_dynamic_shape(mask)
bboxes = odb.tfabsolutely_boxes_to_relative_boxes(bboxes,
width=W,
height=H)
mask = tf.expand_dims(mask, axis=-1)
mask = wmlt.tf_crop_and_resize(mask, bboxes, size=(H, W))
mask = tf.squeeze(mask, axis=-1)
bboxes = odb.restore_sub_area(bboxes,
sub_box=tf.squeeze(instances[RD_BOXES],
axis=0))
probability = tf.squeeze(instances[RD_PROBABILITY], axis=0)
labels = tf.squeeze(instances[RD_LABELS], axis=0)
res_indices = tf.squeeze(instances[RD_INDICES], axis=0)
bboxes_area = odb.box_area(bboxes)
keep_mask = tf.greater(bboxes_area, 1e-3)
bboxes = tf.boolean_mask(bboxes, keep_mask)
labels = tf.boolean_mask(labels, keep_mask)
probability = tf.boolean_mask(probability, keep_mask)
res_indices = tf.boolean_mask(res_indices, keep_mask)
mask = tf.boolean_mask(mask, keep_mask)
bboxes, labels, indices = nms(bboxes, labels, confidence=probability)
probability = tf.gather(probability, indices)
res_indices = tf.gather(res_indices, indices)
mask = tf.gather(mask, indices)
instances[RD_BOXES] = tf.expand_dims(bboxes, axis=0)
instances[RD_MASKS] = tf.expand_dims(mask, axis=0)
instances[RD_LABELS] = tf.expand_dims(labels, axis=0)
instances[RD_PROBABILITY] = tf.expand_dims(probability, axis=0)
instances[RD_INDICES] = tf.expand_dims(res_indices, axis=0)
instances[RD_LENGTH] = tf.shape(labels)
def test_fcos_boxes_encode(self):
box = np.array([[[0.25, 0.25, 0.75, 0.75], [0.1, 0.1, 0.3, 0.3]]],
dtype=np.float32)
labels = np.array([[1, 2]], dtype=np.int32)
length = np.array([2], dtype=np.int32)
fm_shape = [32, 16]
img_size = [224, 112]
with self.test_session() as sess:
regression, center_ness, gt_boxes, classes = fcos_boxes_encode(
min_size=0,
max_size=128,
fm_shape=fm_shape,
img_size=img_size,
gbboxes=box,
glabels=labels,
glength=length)
regression = wmlt.PrintSummary(regression, "regression")
boxes = self.apply_deltas(regression, img_size=img_size)
mask = tf.greater(classes, 0)
boxes = tf.boolean_mask(boxes, mask)
boxes = odbox.tfabsolutely_boxes_to_relative_boxes(
boxes,
width=tf.convert_to_tensor(img_size[1]),
height=tf.convert_to_tensor(img_size[0]))
regression = tf.Print(regression, ["R", regression],
summarize=1000)
regression = tf.Print(regression, ["C", classes], summarize=1000)
regression, center_ness, gt_boxes, classes, boxes = sess.run(
[regression, center_ness, gt_boxes, classes, boxes])
#plt.figure(figsize=(10,10))
#plt.imshow(regression[0,:,:,3])
#plt.imshow(gt_boxes[0,:,:,1])
#plt.imshow(classes[0,:,:])
#plt.show()
print(regression, center_ness, gt_boxes, classes)
wmlu.show_nparray(boxes)
def get_box_in_a_single_layer(self, datas, num_dets, img_size, K):
'''
'''
#wsummary.variable_summaries_v2(datas['heatmaps_tl'],"hm_tl")
h_tl = tf.nn.sigmoid(datas['heatmaps_tl'])
h_br = tf.nn.sigmoid(datas['heatmaps_br'])
h_ct = tf.nn.sigmoid(datas['heatmaps_ct'])
#wsummary.variable_summaries_v2(h_tl,"hm_a_tl")
B, H, W, C = wmlt.combined_static_and_dynamic_shape(h_tl)
h_tl = self.pixel_nms(h_tl)
h_br = self.pixel_nms(h_br)
h_ct = self.pixel_nms(h_ct)
tl_scores, tl_inds, tl_clses, tl_ys, tl_xs = self._topk(h_tl, K=K)
br_scores, br_inds, br_clses, br_ys, br_xs = self._topk(h_br, K=K)
ct_scores, ct_inds, ct_clses, ct_ys, ct_xs = self._topk(h_ct, K=K)
tl_ys = tf.tile(tf.reshape(tl_ys, [B, K, 1]), [1, 1, K])
tl_xs = tf.tile(tf.reshape(tl_xs, [B, K, 1]), [1, 1, K])
br_ys = tf.tile(tf.reshape(br_ys, [B, 1, K]), [1, K, 1])
br_xs = tf.tile(tf.reshape(br_xs, [B, 1, K]), [1, K, 1])
ct_ys = tf.reshape(ct_ys, [B, K])
ct_xs = tf.reshape(ct_xs, [B, K])
ct_scores = tf.reshape(ct_scores, [B, K])
if 'offset_tl' in datas:
tl_regr = wmlt.batch_gather(datas['offset_tl'], tl_inds)
br_regr = wmlt.batch_gather(datas['offset_br'], br_inds)
ct_regr = wmlt.batch_gather(datas['offset_ct'], br_inds)
tl_regr = tf.reshape(tl_regr, [B, K, 1, 2])
br_regr = tf.reshape(br_regr, [B, 1, K, 2])
ct_regr = tf.reshape(ct_regr, [B, K, 2])
tl_xs = tl_xs + tl_regr[..., 0]
tl_ys = tl_ys + tl_regr[..., 1]
br_xs = br_xs + br_regr[..., 0]
br_ys = br_ys + br_regr[..., 1]
ct_xs = ct_xs + ct_regr[..., 0]
ct_ys = ct_ys + ct_regr[..., 1]
bboxes = tf.stack([tl_ys, tl_xs, br_ys, br_xs], axis=-1)
#bboxes = tf.Print(bboxes,["box0",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100)
#wsummary.detection_image_summary(self.inputs[IMAGE],
#boxes=odbox.tfabsolutely_boxes_to_relative_boxes(tf.reshape(bboxes,[B,-1,4]),width=W,height=H),
#name="box0")
tl_tag = wmlt.batch_gather(datas['tag_tl'], tl_inds)
br_tag = wmlt.batch_gather(datas['tag_br'], br_inds)
tl_tag = tf.expand_dims(tl_tag, axis=2)
br_tag = tf.expand_dims(br_tag, axis=1)
tl_tag = tf.tile(tl_tag, [1, 1, K, 1])
br_tag = tf.tile(br_tag, [1, K, 1, 1])
dists = tf.abs(tl_tag - br_tag)
dists = tf.squeeze(dists, axis=-1)
dis_inds = (dists > self.dis_threshold)
tl_scores = tf.tile(tf.reshape(tl_scores, [B, K, 1]), [1, 1, K])
br_scores = tf.tile(tf.reshape(br_scores, [B, 1, K]), [1, K, 1])
scores = (tl_scores + br_scores) / 2
tl_clses = tf.tile(tf.reshape(tl_clses, [B, K, 1]), [1, 1, K])
br_clses = tf.tile(tf.reshape(br_clses, [B, 1, K]), [1, K, 1])
cls_inds = tf.not_equal(tl_clses, br_clses)
width_inds = (br_xs < tl_xs)
height_inds = (br_ys < tl_ys)
all_inds = tf.logical_or(cls_inds, dis_inds)
all_inds = tf.logical_or(all_inds, width_inds)
all_inds = tf.logical_or(all_inds, height_inds)
#all_inds = cls_inds
scores = tf.where(all_inds, tf.zeros_like(scores), scores)
scores, inds = tf.nn.top_k(tf.reshape(scores, [B, -1]), num_dets)
wsummary.variable_summaries_v2(scores, "scores")
wsummary.variable_summaries_v2(tl_scores, "tl_scores")
wsummary.variable_summaries_v2(br_scores, "br_scores")
bboxes = tf.reshape(bboxes, [B, -1, 4])
bboxes = wmlt.batch_gather(bboxes, inds)
#bboxes = tf.Print(bboxes,["box1",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100)
#wsummary.detection_image_summary(self.inputs[IMAGE],
# boxes=odbox.tfabsolutely_boxes_to_relative_boxes(tf.reshape(bboxes,[B,-1,4]),width=W,height=H),
# name="box1")
clses = tf.reshape(tl_clses, [B, -1])
clses = wmlt.batch_gather(clses, inds)
'''tl_scores = tf.reshape(tl_scores,[B,-1,1])
tl_scores = wmlt.batch_gather(tl_scores,inds)
br_scores = tf.reshape(br_scores,[B,-1,1])
br_scores = wmlt.batch_gather(br_scores,inds)'''
ct = tf.stack([ct_ys / tf.to_float(H), ct_xs / tf.to_float(W)],
axis=-1)
bboxes = odbox.tfabsolutely_boxes_to_relative_boxes(bboxes,
width=W,
height=H)
sizes = tf.convert_to_tensor(self.size_threshold, dtype=tf.float32)
relative_size = sizes * tf.rsqrt(
tf.cast(img_size[0] * img_size[1], tf.float32))
_, box_nr, _ = wmlt.combined_static_and_dynamic_shape(bboxes)
length = tf.ones([B], tf.int32) * box_nr
#bboxes = tf.Print(bboxes,["bboxes",tf.reduce_min(bboxes),tf.reduce_max(bboxes),tf.reduce_min(ct),tf.reduce_max(ct)],summarize=100)
center_index = tfop.center_boxes_filter(bboxes=bboxes,
bboxes_clses=clses,
center_points=ct,
center_clses=ct_clses,
size_threshold=relative_size,
bboxes_length=length,
nrs=[3, 5])
def fn(bboxes, scores, clses, ct_score, c_index):
ct_score = tf.gather(ct_score, tf.nn.relu(c_index))
scores = (scores * 2 + ct_score) / 3 #变成三个点的平均
mask = tf.logical_and(tf.greater_equal(c_index, 0),
tf.greater(scores, self.score_threshold))
mask = tf.logical_and(tf.greater_equal(ct_score, 0.001), mask)
bboxes = tf.boolean_mask(bboxes, mask)
scores = tf.boolean_mask(scores, mask)
clses = tf.boolean_mask(clses, mask)
len = tf.reduce_sum(tf.cast(mask, tf.int32))
bboxes = tf.pad(bboxes, [[0, box_nr - len], [0, 0]])
scores = tf.pad(scores, [[0, box_nr - len]])
clses = tf.pad(clses, [[0, box_nr - len]])
return bboxes, scores, clses, len
bboxes, scores, clses, length = tf.map_fn(
lambda x: fn(x[0], x[1], x[2], x[3], x[4]),
elems=(bboxes, scores, clses, ct_scores, center_index),
dtype=(tf.float32, tf.float32, tf.int32, tf.int32))
#bboxes = tf.Print(bboxes,["box2",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100)
#wsummary.detection_image_summary(self.inputs[IMAGE],
# boxes=tf.reshape(bboxes,[B,-1,4]),lengths=length,
# name="box2")
return bboxes, scores, clses, length
def inference_single_image(self,
box_cls,
box_regression,
center_ness,
fm_sizes,
nms=None,
pad=True):
"""
Single-image inference. Return bounding-box detection results by thresholding
on scores and applying non-maximum suppression (NMS).
Arguments:
box_cls;[WxHxA(concat),K]
box_delta [WxHxA(concat),box_dim]
anchors [WxHxA(concat),box_dim]
anchors_size: anchors'size per level
Returns:
Same as `inference`, but for only one image.
"""
assert len(box_cls.get_shape()) == 2, "Error box cls tensor shape"
assert len(
box_regression.get_shape()) == 2, "Error box delta tensor shape"
boxes_all = []
scores_all = []
class_idxs_all = []
# Iterate over every feature level
sizes = [tf.reduce_prod(x) for x in fm_sizes]
box_cls = tf.split(box_cls, num_or_size_splits=sizes)
box_regression = tf.split(box_regression, num_or_size_splits=sizes)
center_ness = tf.split(center_ness, num_or_size_splits=sizes)
img_size = tf.shape(self.batched_inputs[IMAGE])[1:3]
for fm_size, box_cls_i, box_reg_i, centern_ness_i in zip(
fm_sizes, box_cls, box_regression, center_ness):
boxes_i = self.box2box_transform.apply_deltas(regression=box_reg_i,
img_size=img_size,
fm_size=fm_size)
boxes_i = odbox.tfabsolutely_boxes_to_relative_boxes(
boxes_i, width=img_size[1], height=img_size[0])
# (HxWxAxK,)
center_ness_i = tf.reshape(tf.nn.sigmoid(centern_ness_i), [-1, 1])
box_cls_i = tf.nn.sigmoid(
tf.reshape(box_cls_i, [-1, self.num_classes])) * center_ness_i
box_cls_i = tf.reshape(box_cls_i, [-1])
# Keep top k top scoring indices only.
num_topk = tf.minimum(self.topk_candidates, tf.shape(box_reg_i)[0])
# torch.sort is actually faster than .topk (at least on GPUs)
predicted_prob, topk_idxs = tf.nn.top_k(box_cls_i, num_topk)
predicted_prob = predicted_prob[:num_topk]
topk_idxs = topk_idxs[:num_topk]
if self.score_threshold > 1e-5:
keep_idxs = tf.greater(predicted_prob, self.score_threshold)
predicted_prob = tf.boolean_mask(predicted_prob, keep_idxs)
topk_idxs = tf.boolean_mask(topk_idxs, keep_idxs)
boxes_idxs = topk_idxs // self.num_classes
classes_idxs = topk_idxs % self.num_classes
# filter out the proposals with low confidence score
boxes_i = tf.gather(boxes_i, boxes_idxs)
boxes_all.append(boxes_i)
scores_all.append(predicted_prob)
class_idxs_all.append(classes_idxs)
boxes_all, scores_all, class_idxs_all = [
tf.concat(x, axis=0)
for x in [boxes_all, scores_all, class_idxs_all]
]
x, y = wmlt.sort_data(key=scores_all,
datas=[boxes_all, class_idxs_all])
boxes_all, class_idxs_all = y
scores_all, _ = x
if nms is None:
nms = functools.partial(tfop.boxes_nms,
threshold=self.nms_threshold,
classes_wise=True,
k=self.max_detections_per_image)
boxes, labels, nms_idxs = nms(bboxes=boxes_all, classes=class_idxs_all)
scores = tf.gather(scores_all, nms_idxs)
candiate_nr = self.max_detections_per_image
#labels = tf.Print(labels,[tf.shape(labels)],name="XXXXXXXXXXXXXXXXXXXX",summarize=100)
labels = labels + 1 #加上背景
lens = tf.shape(labels)[0]
if pad:
boxes = tf.pad(boxes, paddings=[[0, candiate_nr - lens], [0, 0]])
labels = tf.pad(labels, paddings=[[0, candiate_nr - lens]])
scores = tf.pad(scores, paddings=[[0, candiate_nr - lens]])
return [boxes, labels, scores, lens]
def losses(self):
"""
Args:
For `gt_classes` and `gt_anchors_deltas` parameters, see
:meth:`FCOSGIou.get_ground_truth`.
Their shapes are (N, R) and (N, R, 4), respectively, where R is
the total number of anchors across levels, i.e. sum(Hi x Wi x A)
For `pred_class_logits` and `pred_anchor_deltas`, see
:meth:`FCOSGIouHead.forward`.
Returns:
dict[str: Tensor]:
mapping from a named loss to a scalar tensor
storing the loss. Used during training only. The dict keys are:
"loss_cls" and "loss_box_reg"
"""
assert len(self.pred_logits[0].get_shape()) == 4, "error logits dim"
gt_results = self._get_ground_truth()
loss_cls_list = []
loss_regression_list = []
loss_center_ness_list = []
total_num_foreground = []
img_size = tf.shape(self.batched_inputs[IMAGE])[1:3]
for i, gt_results_item in enumerate(gt_results):
gt_classes = gt_results_item['g_classes']
gt_boxes = gt_results_item['g_boxes']
g_center_ness = gt_results_item['g_center_ness']
pred_class_logits = self.pred_logits[i]
pred_regression = self.pred_regression[i]
pred_center_ness = self.pred_center_ness[i]
foreground_idxs = (gt_classes > 0)
num_foreground = tf.reduce_sum(tf.cast(foreground_idxs, tf.int32))
total_num_foreground.append(num_foreground)
gt_classes_target = tf.one_hot(gt_classes,
depth=self.num_classes + 1)
gt_classes_target = gt_classes_target[..., 1:]
#
pred_center_ness = tf.expand_dims(pred_center_ness, axis=-1)
wsummary.histogram_or_scalar(pred_center_ness, "center_ness")
# logits loss
loss_cls = tf.reduce_sum(
wnn.sigmoid_cross_entropy_with_logits_FL(
labels=gt_classes_target,
logits=pred_class_logits,
alpha=self.focal_loss_alpha,
gamma=self.focal_loss_gamma))
# regression loss
pred_boxes = self.box2box_transform.apply_deltas(
regression=pred_regression, img_size=img_size)
if global_cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.DEBUG and gt_classes.get_shape(
).as_list()[0] > 1:
log_boxes = self.box2box_transform.apply_deltas(
regression=gt_results_item['g_regression'],
img_size=img_size)
log_boxes = odbox.tfabsolutely_boxes_to_relative_boxes(
log_boxes, width=img_size[1], height=img_size[0])
boxes1 = tf.reshape(log_boxes[1:2], [1, -1, 4])
wsummary.detection_image_summary(
images=self.batched_inputs[IMAGE][1:2],
boxes=boxes1,
name="FCOSGIou_decode_test")
pred_center_ness = tf.boolean_mask(pred_center_ness,
foreground_idxs)
g_center_ness = tf.boolean_mask(g_center_ness, foreground_idxs)
pred_boxes = tf.boolean_mask(pred_boxes, foreground_idxs)
gt_boxes = tf.boolean_mask(gt_boxes, foreground_idxs)
wsummary.histogram_or_scalar(pred_center_ness, "center_ness_pos")
reg_loss_sum = (1.0 - odl.giou(pred_boxes, gt_boxes))
wmlt.variable_summaries_v2(reg_loss_sum, f"giou_loss{i}")
pred_center_ness = tf.squeeze(pred_center_ness, axis=-1)
reg_norm = tf.reduce_sum(g_center_ness) + 1e-5
reg_loss_sum = reg_loss_sum * g_center_ness
wmlt.variable_summaries_v2(reg_loss_sum, f"loss_sum{i}")
loss_box_reg = tf.reduce_sum(reg_loss_sum) * 300 / reg_norm
wmlt.variable_summaries_v2(loss_box_reg, f"box_reg_loss_{i}")
loss_center_ness = 0.5 * tf.nn.sigmoid_cross_entropy_with_logits(
labels=g_center_ness, logits=pred_center_ness)
loss_center_ness = tf.reduce_sum(loss_center_ness) * 0.1
wmlt.variable_summaries_v2(loss_center_ness,
f"center_ness_loss{i}")
loss_cls_list.append(loss_cls)
loss_regression_list.append(loss_box_reg)
loss_center_ness_list.append(loss_center_ness)
total_num_foreground = tf.to_float(
tf.maximum(tf.add_n(total_num_foreground), 1))
return {
"fcos_loss_cls":
tf.add_n(loss_cls_list) / total_num_foreground,
"fcos_loss_center_ness":
tf.add_n(loss_center_ness_list) / total_num_foreground,
"fcos_loss_box_reg":
tf.add_n(loss_regression_list) / total_num_foreground
}