def pyapply_deltas(self, datas, img_size=None): ''' ''' h_ct = tf.nn.sigmoid(datas['heatmaps_ct']) offset = datas['offset'] hw = datas['hw'] B, H, W, C = wmlt.combined_static_and_dynamic_shape(h_ct) offset = tf.reshape(offset, [B, -1, 2]) hw = tf.reshape(hw, [B, -1, 2]) h_ct = self.pixel_nms(h_ct, threshold=self.score_threshold) ct_scores, ct_inds, ct_clses, ct_ys, ct_xs = self._topk(h_ct, k=self.k) C = btf.channel(h_ct) hw_inds = ct_inds // C K = self.k ct_ys = tf.reshape(ct_ys, [B, K]) ct_xs = tf.reshape(ct_xs, [B, K]) offset = wmlt.batch_gather(offset, hw_inds) offset = tf.reshape(offset, [B, K, 2]) offset_y, offset_x = tf.unstack(offset, axis=-1) ct_xs = ct_xs + offset_x ct_ys = ct_ys + offset_y hw = wmlt.batch_gather(hw, hw_inds) hw = tf.reshape(hw, [B, K, 2]) h, w = tf.unstack(hw, axis=-1) ymin, xmin, ymax, xmax = [ ct_ys - h / 2, ct_xs - w / 2, ct_ys + h / 2, ct_xs + w / 2 ] bboxes = tf.stack([ymin, xmin, ymax, xmax], axis=-1) bboxes = odb.tfabsolutely_boxes_to_relative_boxes(bboxes, width=W, height=H) return bboxes, ct_clses, ct_scores, hw_inds
def merge_bboxes_and_mask_results(self, instances): nms = functools.partial(tfop.boxes_nms, threshold=self.test_nms_thresh, classes_wise=True) mask = tf.squeeze(instances[RD_MASKS], axis=0) bmask = tf.expand_dims(mask, axis=-1) bmask = wnnl.min_pool2d(bmask, kernel_size=2, stride=1, padding="SAME") bmask = slim.max_pool2d(bmask, kernel_size=2, stride=1, padding="SAME") bmask = tf.squeeze(bmask, axis=-1) bmask = tf.cast(bmask * 255, tf.uint8) #bboxes = tf.squeeze(instances[RD_BOXES],axis=0) bboxes = tfop.get_bboxes_from_mask(bmask) #bboxes = tf.Print(bboxes,["xbboxes",tf.reduce_max(xboxes,keepdims=False), #tf.reduce_max(mask)],summarize=100) Nr, H, W = btf.combined_static_and_dynamic_shape(mask) bboxes = odb.tfabsolutely_boxes_to_relative_boxes(bboxes, width=W, height=H) mask = tf.expand_dims(mask, axis=-1) mask = wmlt.tf_crop_and_resize(mask, bboxes, size=(H, W)) mask = tf.squeeze(mask, axis=-1) bboxes = odb.restore_sub_area(bboxes, sub_box=tf.squeeze(instances[RD_BOXES], axis=0)) probability = tf.squeeze(instances[RD_PROBABILITY], axis=0) labels = tf.squeeze(instances[RD_LABELS], axis=0) res_indices = tf.squeeze(instances[RD_INDICES], axis=0) bboxes_area = odb.box_area(bboxes) keep_mask = tf.greater(bboxes_area, 1e-3) bboxes = tf.boolean_mask(bboxes, keep_mask) labels = tf.boolean_mask(labels, keep_mask) probability = tf.boolean_mask(probability, keep_mask) res_indices = tf.boolean_mask(res_indices, keep_mask) mask = tf.boolean_mask(mask, keep_mask) bboxes, labels, indices = nms(bboxes, labels, confidence=probability) probability = tf.gather(probability, indices) res_indices = tf.gather(res_indices, indices) mask = tf.gather(mask, indices) instances[RD_BOXES] = tf.expand_dims(bboxes, axis=0) instances[RD_MASKS] = tf.expand_dims(mask, axis=0) instances[RD_LABELS] = tf.expand_dims(labels, axis=0) instances[RD_PROBABILITY] = tf.expand_dims(probability, axis=0) instances[RD_INDICES] = tf.expand_dims(res_indices, axis=0) instances[RD_LENGTH] = tf.shape(labels)
def test_fcos_boxes_encode(self): box = np.array([[[0.25, 0.25, 0.75, 0.75], [0.1, 0.1, 0.3, 0.3]]], dtype=np.float32) labels = np.array([[1, 2]], dtype=np.int32) length = np.array([2], dtype=np.int32) fm_shape = [32, 16] img_size = [224, 112] with self.test_session() as sess: regression, center_ness, gt_boxes, classes = fcos_boxes_encode( min_size=0, max_size=128, fm_shape=fm_shape, img_size=img_size, gbboxes=box, glabels=labels, glength=length) regression = wmlt.PrintSummary(regression, "regression") boxes = self.apply_deltas(regression, img_size=img_size) mask = tf.greater(classes, 0) boxes = tf.boolean_mask(boxes, mask) boxes = odbox.tfabsolutely_boxes_to_relative_boxes( boxes, width=tf.convert_to_tensor(img_size[1]), height=tf.convert_to_tensor(img_size[0])) regression = tf.Print(regression, ["R", regression], summarize=1000) regression = tf.Print(regression, ["C", classes], summarize=1000) regression, center_ness, gt_boxes, classes, boxes = sess.run( [regression, center_ness, gt_boxes, classes, boxes]) #plt.figure(figsize=(10,10)) #plt.imshow(regression[0,:,:,3]) #plt.imshow(gt_boxes[0,:,:,1]) #plt.imshow(classes[0,:,:]) #plt.show() print(regression, center_ness, gt_boxes, classes) wmlu.show_nparray(boxes)
def get_box_in_a_single_layer(self, datas, num_dets, img_size, K): ''' ''' #wsummary.variable_summaries_v2(datas['heatmaps_tl'],"hm_tl") h_tl = tf.nn.sigmoid(datas['heatmaps_tl']) h_br = tf.nn.sigmoid(datas['heatmaps_br']) h_ct = tf.nn.sigmoid(datas['heatmaps_ct']) #wsummary.variable_summaries_v2(h_tl,"hm_a_tl") B, H, W, C = wmlt.combined_static_and_dynamic_shape(h_tl) h_tl = self.pixel_nms(h_tl) h_br = self.pixel_nms(h_br) h_ct = self.pixel_nms(h_ct) tl_scores, tl_inds, tl_clses, tl_ys, tl_xs = self._topk(h_tl, K=K) br_scores, br_inds, br_clses, br_ys, br_xs = self._topk(h_br, K=K) ct_scores, ct_inds, ct_clses, ct_ys, ct_xs = self._topk(h_ct, K=K) tl_ys = tf.tile(tf.reshape(tl_ys, [B, K, 1]), [1, 1, K]) tl_xs = tf.tile(tf.reshape(tl_xs, [B, K, 1]), [1, 1, K]) br_ys = tf.tile(tf.reshape(br_ys, [B, 1, K]), [1, K, 1]) br_xs = tf.tile(tf.reshape(br_xs, [B, 1, K]), [1, K, 1]) ct_ys = tf.reshape(ct_ys, [B, K]) ct_xs = tf.reshape(ct_xs, [B, K]) ct_scores = tf.reshape(ct_scores, [B, K]) if 'offset_tl' in datas: tl_regr = wmlt.batch_gather(datas['offset_tl'], tl_inds) br_regr = wmlt.batch_gather(datas['offset_br'], br_inds) ct_regr = wmlt.batch_gather(datas['offset_ct'], br_inds) tl_regr = tf.reshape(tl_regr, [B, K, 1, 2]) br_regr = tf.reshape(br_regr, [B, 1, K, 2]) ct_regr = tf.reshape(ct_regr, [B, K, 2]) tl_xs = tl_xs + tl_regr[..., 0] tl_ys = tl_ys + tl_regr[..., 1] br_xs = br_xs + br_regr[..., 0] br_ys = br_ys + br_regr[..., 1] ct_xs = ct_xs + ct_regr[..., 0] ct_ys = ct_ys + ct_regr[..., 1] bboxes = tf.stack([tl_ys, tl_xs, br_ys, br_xs], axis=-1) #bboxes = tf.Print(bboxes,["box0",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100) #wsummary.detection_image_summary(self.inputs[IMAGE], #boxes=odbox.tfabsolutely_boxes_to_relative_boxes(tf.reshape(bboxes,[B,-1,4]),width=W,height=H), #name="box0") tl_tag = wmlt.batch_gather(datas['tag_tl'], tl_inds) br_tag = wmlt.batch_gather(datas['tag_br'], br_inds) tl_tag = tf.expand_dims(tl_tag, axis=2) br_tag = tf.expand_dims(br_tag, axis=1) tl_tag = tf.tile(tl_tag, [1, 1, K, 1]) br_tag = tf.tile(br_tag, [1, K, 1, 1]) dists = tf.abs(tl_tag - br_tag) dists = tf.squeeze(dists, axis=-1) dis_inds = (dists > self.dis_threshold) tl_scores = tf.tile(tf.reshape(tl_scores, [B, K, 1]), [1, 1, K]) br_scores = tf.tile(tf.reshape(br_scores, [B, 1, K]), [1, K, 1]) scores = (tl_scores + br_scores) / 2 tl_clses = tf.tile(tf.reshape(tl_clses, [B, K, 1]), [1, 1, K]) br_clses = tf.tile(tf.reshape(br_clses, [B, 1, K]), [1, K, 1]) cls_inds = tf.not_equal(tl_clses, br_clses) width_inds = (br_xs < tl_xs) height_inds = (br_ys < tl_ys) all_inds = tf.logical_or(cls_inds, dis_inds) all_inds = tf.logical_or(all_inds, width_inds) all_inds = tf.logical_or(all_inds, height_inds) #all_inds = cls_inds scores = tf.where(all_inds, tf.zeros_like(scores), scores) scores, inds = tf.nn.top_k(tf.reshape(scores, [B, -1]), num_dets) wsummary.variable_summaries_v2(scores, "scores") wsummary.variable_summaries_v2(tl_scores, "tl_scores") wsummary.variable_summaries_v2(br_scores, "br_scores") bboxes = tf.reshape(bboxes, [B, -1, 4]) bboxes = wmlt.batch_gather(bboxes, inds) #bboxes = tf.Print(bboxes,["box1",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100) #wsummary.detection_image_summary(self.inputs[IMAGE], # boxes=odbox.tfabsolutely_boxes_to_relative_boxes(tf.reshape(bboxes,[B,-1,4]),width=W,height=H), # name="box1") clses = tf.reshape(tl_clses, [B, -1]) clses = wmlt.batch_gather(clses, inds) '''tl_scores = tf.reshape(tl_scores,[B,-1,1]) tl_scores = wmlt.batch_gather(tl_scores,inds) br_scores = tf.reshape(br_scores,[B,-1,1]) br_scores = wmlt.batch_gather(br_scores,inds)''' ct = tf.stack([ct_ys / tf.to_float(H), ct_xs / tf.to_float(W)], axis=-1) bboxes = odbox.tfabsolutely_boxes_to_relative_boxes(bboxes, width=W, height=H) sizes = tf.convert_to_tensor(self.size_threshold, dtype=tf.float32) relative_size = sizes * tf.rsqrt( tf.cast(img_size[0] * img_size[1], tf.float32)) _, box_nr, _ = wmlt.combined_static_and_dynamic_shape(bboxes) length = tf.ones([B], tf.int32) * box_nr #bboxes = tf.Print(bboxes,["bboxes",tf.reduce_min(bboxes),tf.reduce_max(bboxes),tf.reduce_min(ct),tf.reduce_max(ct)],summarize=100) center_index = tfop.center_boxes_filter(bboxes=bboxes, bboxes_clses=clses, center_points=ct, center_clses=ct_clses, size_threshold=relative_size, bboxes_length=length, nrs=[3, 5]) def fn(bboxes, scores, clses, ct_score, c_index): ct_score = tf.gather(ct_score, tf.nn.relu(c_index)) scores = (scores * 2 + ct_score) / 3 #变成三个点的平均 mask = tf.logical_and(tf.greater_equal(c_index, 0), tf.greater(scores, self.score_threshold)) mask = tf.logical_and(tf.greater_equal(ct_score, 0.001), mask) bboxes = tf.boolean_mask(bboxes, mask) scores = tf.boolean_mask(scores, mask) clses = tf.boolean_mask(clses, mask) len = tf.reduce_sum(tf.cast(mask, tf.int32)) bboxes = tf.pad(bboxes, [[0, box_nr - len], [0, 0]]) scores = tf.pad(scores, [[0, box_nr - len]]) clses = tf.pad(clses, [[0, box_nr - len]]) return bboxes, scores, clses, len bboxes, scores, clses, length = tf.map_fn( lambda x: fn(x[0], x[1], x[2], x[3], x[4]), elems=(bboxes, scores, clses, ct_scores, center_index), dtype=(tf.float32, tf.float32, tf.int32, tf.int32)) #bboxes = tf.Print(bboxes,["box2",tf.reduce_max(bboxes),tf.reduce_min(bboxes),W,H],summarize=100) #wsummary.detection_image_summary(self.inputs[IMAGE], # boxes=tf.reshape(bboxes,[B,-1,4]),lengths=length, # name="box2") return bboxes, scores, clses, length
def inference_single_image(self, box_cls, box_regression, center_ness, fm_sizes, nms=None, pad=True): """ Single-image inference. Return bounding-box detection results by thresholding on scores and applying non-maximum suppression (NMS). Arguments: box_cls;[WxHxA(concat),K] box_delta [WxHxA(concat),box_dim] anchors [WxHxA(concat),box_dim] anchors_size: anchors'size per level Returns: Same as `inference`, but for only one image. """ assert len(box_cls.get_shape()) == 2, "Error box cls tensor shape" assert len( box_regression.get_shape()) == 2, "Error box delta tensor shape" boxes_all = [] scores_all = [] class_idxs_all = [] # Iterate over every feature level sizes = [tf.reduce_prod(x) for x in fm_sizes] box_cls = tf.split(box_cls, num_or_size_splits=sizes) box_regression = tf.split(box_regression, num_or_size_splits=sizes) center_ness = tf.split(center_ness, num_or_size_splits=sizes) img_size = tf.shape(self.batched_inputs[IMAGE])[1:3] for fm_size, box_cls_i, box_reg_i, centern_ness_i in zip( fm_sizes, box_cls, box_regression, center_ness): boxes_i = self.box2box_transform.apply_deltas(regression=box_reg_i, img_size=img_size, fm_size=fm_size) boxes_i = odbox.tfabsolutely_boxes_to_relative_boxes( boxes_i, width=img_size[1], height=img_size[0]) # (HxWxAxK,) center_ness_i = tf.reshape(tf.nn.sigmoid(centern_ness_i), [-1, 1]) box_cls_i = tf.nn.sigmoid( tf.reshape(box_cls_i, [-1, self.num_classes])) * center_ness_i box_cls_i = tf.reshape(box_cls_i, [-1]) # Keep top k top scoring indices only. num_topk = tf.minimum(self.topk_candidates, tf.shape(box_reg_i)[0]) # torch.sort is actually faster than .topk (at least on GPUs) predicted_prob, topk_idxs = tf.nn.top_k(box_cls_i, num_topk) predicted_prob = predicted_prob[:num_topk] topk_idxs = topk_idxs[:num_topk] if self.score_threshold > 1e-5: keep_idxs = tf.greater(predicted_prob, self.score_threshold) predicted_prob = tf.boolean_mask(predicted_prob, keep_idxs) topk_idxs = tf.boolean_mask(topk_idxs, keep_idxs) boxes_idxs = topk_idxs // self.num_classes classes_idxs = topk_idxs % self.num_classes # filter out the proposals with low confidence score boxes_i = tf.gather(boxes_i, boxes_idxs) boxes_all.append(boxes_i) scores_all.append(predicted_prob) class_idxs_all.append(classes_idxs) boxes_all, scores_all, class_idxs_all = [ tf.concat(x, axis=0) for x in [boxes_all, scores_all, class_idxs_all] ] x, y = wmlt.sort_data(key=scores_all, datas=[boxes_all, class_idxs_all]) boxes_all, class_idxs_all = y scores_all, _ = x if nms is None: nms = functools.partial(tfop.boxes_nms, threshold=self.nms_threshold, classes_wise=True, k=self.max_detections_per_image) boxes, labels, nms_idxs = nms(bboxes=boxes_all, classes=class_idxs_all) scores = tf.gather(scores_all, nms_idxs) candiate_nr = self.max_detections_per_image #labels = tf.Print(labels,[tf.shape(labels)],name="XXXXXXXXXXXXXXXXXXXX",summarize=100) labels = labels + 1 #加上背景 lens = tf.shape(labels)[0] if pad: boxes = tf.pad(boxes, paddings=[[0, candiate_nr - lens], [0, 0]]) labels = tf.pad(labels, paddings=[[0, candiate_nr - lens]]) scores = tf.pad(scores, paddings=[[0, candiate_nr - lens]]) return [boxes, labels, scores, lens]
def losses(self): """ Args: For `gt_classes` and `gt_anchors_deltas` parameters, see :meth:`FCOSGIou.get_ground_truth`. Their shapes are (N, R) and (N, R, 4), respectively, where R is the total number of anchors across levels, i.e. sum(Hi x Wi x A) For `pred_class_logits` and `pred_anchor_deltas`, see :meth:`FCOSGIouHead.forward`. Returns: dict[str: Tensor]: mapping from a named loss to a scalar tensor storing the loss. Used during training only. The dict keys are: "loss_cls" and "loss_box_reg" """ assert len(self.pred_logits[0].get_shape()) == 4, "error logits dim" gt_results = self._get_ground_truth() loss_cls_list = [] loss_regression_list = [] loss_center_ness_list = [] total_num_foreground = [] img_size = tf.shape(self.batched_inputs[IMAGE])[1:3] for i, gt_results_item in enumerate(gt_results): gt_classes = gt_results_item['g_classes'] gt_boxes = gt_results_item['g_boxes'] g_center_ness = gt_results_item['g_center_ness'] pred_class_logits = self.pred_logits[i] pred_regression = self.pred_regression[i] pred_center_ness = self.pred_center_ness[i] foreground_idxs = (gt_classes > 0) num_foreground = tf.reduce_sum(tf.cast(foreground_idxs, tf.int32)) total_num_foreground.append(num_foreground) gt_classes_target = tf.one_hot(gt_classes, depth=self.num_classes + 1) gt_classes_target = gt_classes_target[..., 1:] # pred_center_ness = tf.expand_dims(pred_center_ness, axis=-1) wsummary.histogram_or_scalar(pred_center_ness, "center_ness") # logits loss loss_cls = tf.reduce_sum( wnn.sigmoid_cross_entropy_with_logits_FL( labels=gt_classes_target, logits=pred_class_logits, alpha=self.focal_loss_alpha, gamma=self.focal_loss_gamma)) # regression loss pred_boxes = self.box2box_transform.apply_deltas( regression=pred_regression, img_size=img_size) if global_cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.DEBUG and gt_classes.get_shape( ).as_list()[0] > 1: log_boxes = self.box2box_transform.apply_deltas( regression=gt_results_item['g_regression'], img_size=img_size) log_boxes = odbox.tfabsolutely_boxes_to_relative_boxes( log_boxes, width=img_size[1], height=img_size[0]) boxes1 = tf.reshape(log_boxes[1:2], [1, -1, 4]) wsummary.detection_image_summary( images=self.batched_inputs[IMAGE][1:2], boxes=boxes1, name="FCOSGIou_decode_test") pred_center_ness = tf.boolean_mask(pred_center_ness, foreground_idxs) g_center_ness = tf.boolean_mask(g_center_ness, foreground_idxs) pred_boxes = tf.boolean_mask(pred_boxes, foreground_idxs) gt_boxes = tf.boolean_mask(gt_boxes, foreground_idxs) wsummary.histogram_or_scalar(pred_center_ness, "center_ness_pos") reg_loss_sum = (1.0 - odl.giou(pred_boxes, gt_boxes)) wmlt.variable_summaries_v2(reg_loss_sum, f"giou_loss{i}") pred_center_ness = tf.squeeze(pred_center_ness, axis=-1) reg_norm = tf.reduce_sum(g_center_ness) + 1e-5 reg_loss_sum = reg_loss_sum * g_center_ness wmlt.variable_summaries_v2(reg_loss_sum, f"loss_sum{i}") loss_box_reg = tf.reduce_sum(reg_loss_sum) * 300 / reg_norm wmlt.variable_summaries_v2(loss_box_reg, f"box_reg_loss_{i}") loss_center_ness = 0.5 * tf.nn.sigmoid_cross_entropy_with_logits( labels=g_center_ness, logits=pred_center_ness) loss_center_ness = tf.reduce_sum(loss_center_ness) * 0.1 wmlt.variable_summaries_v2(loss_center_ness, f"center_ness_loss{i}") loss_cls_list.append(loss_cls) loss_regression_list.append(loss_box_reg) loss_center_ness_list.append(loss_center_ness) total_num_foreground = tf.to_float( tf.maximum(tf.add_n(total_num_foreground), 1)) return { "fcos_loss_cls": tf.add_n(loss_cls_list) / total_num_foreground, "fcos_loss_center_ness": tf.add_n(loss_center_ness_list) / total_num_foreground, "fcos_loss_box_reg": tf.add_n(loss_regression_list) / total_num_foreground }