def forward(self, inputs, outputs): """See modeling.detector.CollectAndDistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is # [rpn_rois_fpn2, ..., rpn_rois_fpn6, # rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6] # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] rois = collect(inputs, self._train) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs): """See modeling.detector.CollectAndDistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is # [rpn_rois_fpn2, ..., rpn_rois_fpn6, # rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6] # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] rois = collect(inputs, self._train) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_info, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs): """See modeling.detector.GenerateProposalLabels for inputs/outputs documentation. """ # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] rois = inputs[0].data roidb = blob_utils.deserialize(inputs[1].data) im_info = inputs[2].data im_scales = im_info[:, 2] if cfg.POLYGON.POLYGON_ON: output_blob_names = polygon_rcnn_roi_data.get_polygon_rcnn_blob_names( ) else: output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) if cfg.POLYGON.POLYGON_ON: roidb_utils.add_polygon_regression_targets(roidb) else: roidb_utils.add_bbox_regression_targets(roidb) blobs = {k: [] for k in output_blob_names} if cfg.POLYGON.POLYGON_ON: polygon_rcnn_roi_data.add_polygon_rcnn_blobs( blobs, im_scales, roidb) else: fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def get_minibatch_s6(roidb, roidb_noclass): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list if 0: random_bbox = dict() random_bbox['kernel_size'] = 224 random_bbox['tl_x'] = random.randint(0, 800) random_bbox['tl_y'] = random.randint(0, 800) blobs = {k: [] for k in get_minibatch_blob_names()} # Get the input image blob, formatted for caffe2 im_blob, im_scales, error_flag = _get_image_blob_s6(roidb, roidb_noclass) blobs['data'] = im_blob if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb, im_width, im_height) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) return blobs, valid
def get_minibatch_mul(roidb): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list blobs = {k: [] for k in get_minibatch_blob_names()} #print("minibatch.py 157 ---------{}".format(len(roidb))) # Get the input image blob, formatted for caffe2 im_blob, im_scales, roidb_ = _get_img_blob_mul(roidb) blobs['data'] = im_blob if cfg.RPN.RPN_ON: # # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb_) return blobs, valid if cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb_, im_width, im_height) else: # Fast R-CNN like models trained on # precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb_) return blobs, valid
def get_minibatch(roidb): """载入roidb,构建一个minibatch采样.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list # 生成空字典数据 如 {'data': [], 'fpn_labels_int32_wide_fpn1': []}等 #print("现在执行的是无扩增版本") blobs = {k: [] for k in get_minibatch_blob_names()} print("blobs is ", blobs) # Get the input image blob, formatted for caffe2 im_blob, im_scales = _get_image_blob(roidb) blobs['data'] = im_blob if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb, im_width, im_height) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) return blobs, valid
def get_minibatch(roidb): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list blobs = {k: [] for k in get_minibatch_blob_names()} # Get the input image blob, formatted for caffe2 im_blob, im_scales = _get_image_blob(roidb) blobs['data'] = im_blob # Calculate tracking metadata if using pre-calculated blobs if len(cfg.DATA_LOADER.EXTRA_BLOBS): blobs_extra = _get_extra_blobs(roidb) blobs.update(blobs_extra) if 'track_n_rois' in cfg.DATA_LOADER.EXTRA_BLOBS: blobs.update({ 'track_n_rois_one': [blobs_extra['track_n_rois'][0]], 'track_n_rois_two': [blobs_extra['track_n_rois'][1]], }) if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb, im_width, im_height) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) return blobs, valid
def get_minibatch(roidb): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list blobs = {k: [] for k in get_minibatch_blob_names()} # Get the input image blob, formatted for caffe2 im_blob, im_scales, im_crops = _get_image_blob(roidb) blobs['data'] = im_blob # row col row col to x1 y1 x2 y2 im_crops = np.array(im_crops, dtype=np.int32) im_crops = im_crops[:, (1, 0, 3, 2)] if cfg.REID.APM: valid = reid_apm_roi_data.add_reid_blobs(blobs, im_scales, im_crops, roidb) else: valid = reid_roi_data.add_reid_blobs(blobs, im_scales, roidb) return blobs, valid if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb, im_width, im_height) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) return blobs, valid
def forward(self, inputs, outputs): rois, transfer_rois = collect(inputs, self._train, self._mc) im_info = inputs[-1].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2].data) json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb, transfer_rois) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs): """See modeling.detector.GenerateProposalLabels for inputs/outputs documentation. """ # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] rois = inputs[0].data roidb = blob_utils.deserialize(inputs[1].data) im_info = inputs[2].data im_scales = im_info[:, 2] output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names() # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs): """See modeling.detector.DistributeFpnRpnProposals for inputs/outputs documentation. """ # inputs is [rois] out from decode_bbox operator # If training with Faster R-CNN, then inputs will additionally include # + [roidb, im_info] _rois = inputs[0].data rois = remove_invalid_boxes(_rois, self._stage_num) # print('++++++++++++++++ DFRP Op of RCNN stage {} ++++++++++++++++++'.format(self._stage_num)) if self._train: # During training we reuse the data loader code. We populate roidb # entries on the fly using the rois generated by RPN. # im_info: [[im_height, im_width, im_scale], ...] im_info = inputs[2].data im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[1].data) json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_cascade_fast_rcnn_blob_names( is_training=True, stage_num=self._stage_num) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb, self._stage_num) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) # reset roidb for next rcnn stage, remove 'max_overlaps', 'max_classes', 'bbox_targets' in each roidb, # intialize 'boxes', 'seg_areas', 'gt_classes', 'gt_overlaps', 'box_to_gt_ind_map' only contain gt infos # if self._stage_num == 2: # json_dataset.reset_roidb_for_next_stage(roidb) else: # For inference we have a special code path that avoids some data # loader overhead distribute(rois, None, outputs, self._train, self._stage_num)
def get_minibatch(roidb): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list blobs = {k: [] for k in get_minibatch_blob_names()} # Get the input image blob, formatted for caffe2 im_blob, im_scales = _get_image_blob(roidb) blobs['data'] = im_blob if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs( blobs, im_scales, roidb, im_width, im_height ) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) return blobs, valid
def get_minibatch(roidb): """Given a roidb, construct a minibatch sampled from it.""" # We collect blobs from each image onto a list and then concat them into a # single tensor, hence we initialize each blob to an empty list blobs = {k: [] for k in get_minibatch_blob_names()} # Get the input image blob, formatted for caffe2 # im_blob, im_scales = _get_image_blob(roidb) im_blob, im_scales, pose_pred, pose_line, blobs['seg_gt_label'] = _get_image_pose_blob(roidb) # pose_pred the same shape with im_blob blobs['data'] = im_blob blobs['normalizer'] = np.array([100], dtype=np.float32) if 'LIP' in cfg.TRAIN.DATASETS[0]: blobs['pose_pred_4'], blobs['pose_pred_8'], blobs['pose_pred_16'], blobs['pose_pred_32'] = _resize_pose_blob(pose_pred, channel=26) else: blobs['pose_pred_4'], blobs['pose_pred_8'], blobs['pose_pred_16'], blobs['pose_pred_32'] = _resize_pose_blob(pose_pred, channel=26) # blobs['pose_pred_8'], blobs['pose_pred_16'] = _resize_pose_blob_to13(pose_pred) # pose 16 to 13 channel # blobs['pose_sum_8'], blobs['pose_sum_16'] = pose_sum_to_onehotmap(blobs['pose_pred_8'], blobs['pose_pred_16']) blobs['pose_line_8'], blobs['pose_line_16'] = _resize_poseline_blob(pose_line) if cfg.RPN.RPN_ON: # RPN-only or end-to-end Faster/Mask R-CNN valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb) elif cfg.RETINANET.RETINANET_ON: im_width, im_height = im_blob.shape[3], im_blob.shape[2] # im_width, im_height corresponds to the network input: padded image # (if needed) width and height. We pass it as input and slice the data # accordingly so that we don't need to use SampleAsOp valid = retinanet_roi_data.add_retinanet_blobs( blobs, im_scales, roidb, im_width, im_height ) else: # Fast R-CNN like models trained on precomputed proposals valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) # blobs['pose_pred'] = pose_pred_model.pred_pose_batch(roidb) # pose_pred_model.draw_batch(blobs['pose_pred'], roidb) # blobs['pose_pred'] = _get_pose_pred(roidb) # logger.info(blobs['pose_pred'].shape) return blobs, valid
def train_model(): """Model training loop.""" logger = logging.getLogger(__name__) model, weights_file, start_iter, checkpoints, output_dir = create_model( ) #for create model if 'final' in checkpoints: # The final model was found in the output directory, so nothing to do return checkpoints if 0: output_dir = '/home/icubic/daily_work/code/Detectron/train/coco_2014_train_ET_PH_part/generalized_rcnn_multi/' #output_dir = output_dir + '_101' setup_model_for_training(model, weights_file, output_dir) training_stats = TrainingStats(model) uuuu = model.roi_data_loader._blobs_queue_name CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS) print('------------train.py') for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER): training_stats.IterTic() lr = model.UpdateWorkspaceLr(cur_iter, lr_policy.get_lr_at_iter(cur_iter)) #aaa_debug = workspace.FetchBlob('gpu_0/data') #bbb_debug = workspace.FetchBlob('gpu_0/conv1_w') #ccc_debug = workspace.FetchBlob('gpu_0/'+uuuu) try: workspace.RunNet(model.net.Proto().name) if 0: #import detectron.utils.blob as blob_utils inputs = [workspace.FetchBlob("gpu_0/rpn_rois_fpn2"),workspace.FetchBlob("gpu_0/rpn_rois_fpn3"),workspace.FetchBlob("gpu_0/rpn_rois_fpn4"),workspace.FetchBlob("gpu_0/rpn_rois_fpn5"), \ workspace.FetchBlob("gpu_0/rpn_rois_fpn6"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn2"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn3"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn4"), \ workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn5"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn6"),workspace.FetchBlob("gpu_0/roidb"),workspace.FetchBlob("gpu_0/im_info"),\ ] rois = collect(inputs, True) #inputs.append(workspace.FetchBlob("gpu_0/rpn_rois_fpn2")) im_info = inputs[-1] im_scales = im_info[:, 2] roidb = blob_utils.deserialize(inputs[-2]) # For historical consistency with the original Faster R-CNN # implementation we are *not* filtering crowd proposals. # This choice should be investigated in the future (it likely does # not matter). json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0) roidb_utils.add_bbox_regression_targets(roidb) # Compute training labels for the RPN proposals; also handles # distributing the proposals over FPN levels output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names( ) blobs = {k: [] for k in output_blob_names} fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i]) #if (np.sum(bb == 1))>0: # print('cc') except: aa = workspace.FetchBlob("gpu_0/rpn_rois_fpn2") aaa_debug = workspace.FetchBlob('gpu_0/data') print('aaaaaerror') #print("blobs:\n{}".format(workspace.Blobs())) #print('train.py aaaaaaaa_debug') if 1: aaa = workspace.FetchBlob("gpu_0/data") # nchw #img = aaa[1].copy() # BGR HWC -> CHW 12 #transform_img = img.swapaxes(0, 1).swapaxes(1, 2) #cv2.imshow("image0 ", transform_img[:, :, (2, 1, 0)]) #cv2.waitKey(0) #cv2.destroyAllWindows() #cv2.imshow('/home/icubic/daily_work/code/Detectron/aaa.png', aaa[0]) aaa_debug = workspace.FetchBlob('gpu_0/data') bbb_debug = workspace.FetchBlob('gpu_0/conv1_w') ccc_debug = workspace.FetchBlob('gpu_0/' + uuuu) ddd_debug = workspace.FetchBlob('gpu_0/roidb') eee_debug = workspace.FetchBlob('gpu_0/im_info') #print("Fetched data:\n{}".format(workspace.FetchBlob("gpu_0/data"))) if cur_iter == start_iter: nu.print_net(model) training_stats.IterToc() training_stats.UpdateIterStats() training_stats.LogIterStats(cur_iter, lr) if (cur_iter + 1) % ( CHECKPOINT_PERIOD / 4 ) == 0 and cur_iter > start_iter: #((cur_iter + 1) % (CHECKPOINT_PERIOD/1) == 0 and (cur_iter > start_iter and cur_iter < 50000)) or ((cur_iter + 1) % (CHECKPOINT_PERIOD/8) == 0 and cur_iter > 50000): checkpoints[cur_iter] = os.path.join( output_dir, 'model_iter_50_{}.pkl'.format(cur_iter)) nu.save_model_to_weights_file(checkpoints[cur_iter], model) if cur_iter == start_iter + training_stats.LOG_PERIOD: # Reset the iteration timer to remove outliers from the first few # SGD iterations training_stats.ResetIterTimer() if np.isnan(training_stats.iter_total_loss): logger.critical('Loss is NaN, exiting...') model.roi_data_loader.shutdown() envu.exit_on_error() # Save the final model checkpoints['final'] = os.path.join(output_dir, 'model_final_50.pkl') nu.save_model_to_weights_file(checkpoints['final'], model) # Shutdown data loading threads model.roi_data_loader.shutdown() return checkpoints