def forward(self, inputs, outputs): """See modeling.detector.GenerateProposalLabels for inputs/outputs documentation. """ # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] rois = inputs[0].data roidb = blob_utils.deserialize(inputs[1].data) im_info = inputs[2].data im_scales = im_info[:, 2] output_blob_names = cascade_rcnn_roi_data.get_cascade_rcnn_stage_3_blob_names( ) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) blobs = {k: [] for k in output_blob_names} cascade_rcnn_roi_data.add_cascade_rcnn_stage_3_blobs( blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def distribute_plus_pose(rois, label_blobs, inputs, outputs, train): lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) im_info = inputs[-1].data # print('inputs[-1].data shape:', im_info.shape) im_info = im_info[0] # print('im_info shape:', im_info.shape) im_scale = im_info[2] im_h = im_info[0] im_w = im_info[1] output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(train) blobs = {k: [] for k in output_blob_names} hg_rois = rois * 1. / im_scale * np.array( [1, 384.0 / im_w, 384.0 / im_h, 384.0 / im_w, 384.0 / im_h], dtype=np.float32) # hg_rois = rois[:, 1:5] * 1. / im_scale * np.array([1, 255.0/im_w, 255.0/im_h, 255.0/im_w, 255.0/im_h], dtype=np.float32) blobs['rois'] = rois blobs['rois_hg'] = hg_rois fpn.add_multilevel_roi_blobs(blobs, 'rois', blobs['rois'], lvls, lvl_min, lvl_max) fpn.add_multilevel_roi_blobs(blobs, 'rois_hg', blobs['rois_hg'], lvls, lvl_min, lvl_max) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs): """See modeling.detector.CollectAndDistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is # [rpn_rois_fpn2, ..., rpn_rois_fpn6, # rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6] # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] rois = collect(inputs, self._train) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_info, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train)
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) # output[0]存储所有的rois outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) # 对fpn的每一层创建新的rois rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): # 选取本层的roi索引 idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) # rois中每一项在重新分配到fpn后的连接数组中的位置 rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def forward(self, inputs, outputs): """See modeling.detector.DistributeCascadeProposals for inputs/outputs documentation. """ rois = inputs[0].data if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] roidb = blob_utils.deserialize(inputs[1].data) im_info = inputs[2].data im_scales = im_info[:, 2] # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = cascade_rcnn_roi_data.get_cascade_rcnn_blob_names( self._stage) blobs = {k: [] for k in output_blob_names} # 进行rois映射到了合适的fpn层, 并重新进行采样构成训练数据 cascade_rcnn_roi_data.add_cascade_rcnn_blobs( blobs, im_scales, roidb, self._stage) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train)
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of roi_data.cascade_rcnn.get_cascade_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level ###对每一个lvl构建新的输出blob # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): ###获取属于该lvl的rois的索引(行数) idx_lvl = np.where(lvls == lvl)[0] ###将属于该lvl的所有rois放进新的blob中 blob_roi_level = rois[idx_lvl, :] ###将该lvl的blob放入最终的输出blob对应位置中 outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) ###这里又是在干嘛没看懂 rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def forward(self, inputs, outputs): """See modeling.detector.CollectAndDistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is # [rpn_rois_fpn2, ..., rpn_rois_fpn6, # rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6] # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] rois = collect(inputs, self._train) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs): """See modeling.detector.GenerateTrackingLabels for inputs/outputs documentation. """ track_ids_one = inputs[0].data track_ids_two = inputs[1].data tracking_labels = np.array([id_one == id_two \ for id_one in track_ids_one for id_two in track_ids_two], dtype=np.int32) n_matches = sum(tracking_labels) assert n_matches > 0, "Image pair with no matches encountered" assert len(tracking_labels) - n_matches > 0, "Image pair with only matches encountered" blob_utils.py_op_copy_blob(tracking_labels, outputs[0])
def forward(self, inputs, outputs): # The inputs contains [bbox_pred, cls_prob, rois] # rois --> np.array((num,5)), (batch_idx, x1, y2, x2, y2) # print('++++++++++++++++++++++ Decode BBox of rcnn stage {} +++++++++++++++++++++++'.format(self._stage_num)) cls_prob = inputs[0].data[...] bbox_pred = inputs[1].data[...] rois = inputs[2].data[...] if self._train: overlaps = inputs[3].data[...] im_info = inputs[4].data else: im_info = inputs[3].data proposals_next = rois[:, 1:5] # Use delta with max cls_score as deltas adding to rois if cfg.MODEL.CLS_AGNOSTIC_BBOX_REG: delta = bbox_pred[:, 4:bbox_pred.shape[1]] else: cls_idx = cls_prob.argmax(axis=1) delta = np.zeros((bbox_pred.shape[0], 4), dtype=bbox_pred.dtype) for i in range(cls_idx.shape[0]): delta[i, :] = bbox_pred[i, cls_idx[i] * 4:cls_idx[i] * 4 + 4] # Add bbox deltas onto rois to generate new rois if self._stage_num == 1: bbox_reg_weights = cfg.CASCADERCNN.BBOX_REG_WEIGHTS_STAGE1 elif self._stage_num == 2: bbox_reg_weights = cfg.CASCADERCNN.BBOX_REG_WEIGHTS_STAGE2 new_rois = box_utils.bbox_transform(proposals_next, delta, bbox_reg_weights) batch_idxs = rois[:, 0].reshape(rois.shape[0], 1) new_rois = np.hstack((batch_idxs, new_rois)) # remove invalid boxes output_rois = remove_invalid_boxes(new_rois) if self._train: # screen out high IOU boxes, to remove redundant gt boxes output_rois = remove_high_iou_boxes(output_rois, overlaps) else: output_rois = output_rois # clip tiled boxes into image output_rois = clip_tiled_bboxes(output_rois, im_info[0, :2]) blob_utils.py_op_copy_blob(output_rois, outputs[0])
def forward(self, inputs, outputs): rois, transfer_rois = collect(inputs, self._train, self._mc) im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb, transfer_rois) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs): """See modeling.detector.GenerateProposalLabels for inputs/outputs documentation. """ # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] rois = inputs[0].data roidb = blob_utils.deserialize(inputs[1].data) im_info = inputs[2].data im_scales = im_info[:, 2] output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs): """See modeling.detector.DistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is [rois] out from decode_bbox operator # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] _rois = inputs[0].data rois = remove_invalid_boxes(_rois, self._stage_num) # print('++++++++++++++++ DFRP Op of RCNN stage {} ++++++++++++++++++'.format(self._stage_num)) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[2].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[1].data) json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_cascade_fast_rcnn_blob_names( is_training=True, stage_num=self._stage_num) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb, self._stage_num) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) # reset roidb for next rcnn stage, remove 'max_overlaps', 'max_classes', 'bbox_targets' in each roidb, # intialize 'boxes', 'seg_areas', 'gt_classes', 'gt_overlaps', 'box_to_gt_ind_map' only contain gt infos # if self._stage_num == 2: # json_dataset.reset_roidb_for_next_stage(roidb) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train, self._stage_num)
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def train_model(): """Model training loop.""" logger = logging.getLogger(__name__) model, weights_file, start_iter, checkpoints, output_dir = create_model( ) #for create model if 'final' in checkpoints: # The final model was found in the output directory, so nothing to do return checkpoints if 0: output_dir = '/home/icubic/daily_work/code/Detectron/train/coco_2014_train_ET_PH_part/generalized_rcnn_multi/' #output_dir = output_dir + '_101' setup_model_for_training(model, weights_file, output_dir) training_stats = TrainingStats(model) uuuu = model.roi_data_loader._blobs_queue_name CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS) print('------------train.py') for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER): training_stats.IterTic() lr = model.UpdateWorkspaceLr(cur_iter, lr_policy.get_lr_at_iter(cur_iter)) #aaa_debug = workspace.FetchBlob('gpu_0/data') #bbb_debug = workspace.FetchBlob('gpu_0/conv1_w') #ccc_debug = workspace.FetchBlob('gpu_0/'+uuuu) try: workspace.RunNet(model.net.Proto().name) if 0: #import detectron.utils.blob as blob_utils inputs = [workspace.FetchBlob("gpu_0/rpn_rois_fpn2"),workspace.FetchBlob("gpu_0/rpn_rois_fpn3"),workspace.FetchBlob("gpu_0/rpn_rois_fpn4"),workspace.FetchBlob("gpu_0/rpn_rois_fpn5"), \ workspace.FetchBlob("gpu_0/rpn_rois_fpn6"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn2"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn3"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn4"), \ workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn5"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn6"),workspace.FetchBlob("gpu_0/roidb"),workspace.FetchBlob("gpu_0/im_info"),\ ] rois = collect(inputs, True) #inputs.append(workspace.FetchBlob("gpu_0/rpn_rois_fpn2")) im_info = inputs[-1] im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2]) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names( ) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) #if (np.sum(bb == 1))>0: # print('cc') except: aa = workspace.FetchBlob("gpu_0/rpn_rois_fpn2") aaa_debug = workspace.FetchBlob('gpu_0/data') print('aaaaaerror') #print("blobs:\n{}".format(workspace.Blobs())) #print('train.py aaaaaaaa_debug') if 1: aaa = workspace.FetchBlob("gpu_0/data") # nchw #img = aaa[1].copy() # BGR HWC -> CHW 12 #transform_img = img.swapaxes(0, 1).swapaxes(1, 2) #cv2.imshow("image0 ", transform_img[:, :, (2, 1, 0)]) #cv2.waitKey(0) #cv2.destroyAllWindows() #cv2.imshow('/home/icubic/daily_work/code/Detectron/aaa.png', aaa[0]) aaa_debug = workspace.FetchBlob('gpu_0/data') bbb_debug = workspace.FetchBlob('gpu_0/conv1_w') ccc_debug = workspace.FetchBlob('gpu_0/' + uuuu) ddd_debug = workspace.FetchBlob('gpu_0/roidb') eee_debug = workspace.FetchBlob('gpu_0/im_info') #print("Fetched data:\n{}".format(workspace.FetchBlob("gpu_0/data"))) if cur_iter == start_iter: nu.print_net(model) training_stats.IterToc() training_stats.UpdateIterStats() training_stats.LogIterStats(cur_iter, lr) if (cur_iter + 1) % ( CHECKPOINT_PERIOD / 4 ) == 0 and cur_iter > start_iter: #((cur_iter + 1) % (CHECKPOINT_PERIOD/1) == 0 and (cur_iter > start_iter and cur_iter < 50000)) or ((cur_iter + 1) % (CHECKPOINT_PERIOD/8) == 0 and cur_iter > 50000): checkpoints[cur_iter] = os.path.join( output_dir, 'model_iter_50_{}.pkl'.format(cur_iter)) nu.save_model_to_weights_file(checkpoints[cur_iter], model) if cur_iter == start_iter + training_stats.LOG_PERIOD: # Reset the iteration timer to remove outliers from the first few # SGD iterations training_stats.ResetIterTimer() if np.isnan(training_stats.iter_total_loss): logger.critical('Loss is NaN, exiting...') model.roi_data_loader.shutdown() envu.exit_on_error() # Save the final model checkpoints['final'] = os.path.join(output_dir, 'model_final_50.pkl') nu.save_model_to_weights_file(checkpoints['final'], model) # Shutdown data loading threads model.roi_data_loader.shutdown() return checkpoints