def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = cascade_rcnn_roi_data.get_cascade_rcnn_stage_3_blob_names(
)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
blobs = {k: [] for k in output_blob_names}
cascade_rcnn_roi_data.add_cascade_rcnn_stage_3_blobs(
blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def distribute_plus_pose(rois, label_blobs, inputs, outputs, train):
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
im_info = inputs[-1].data
# print('inputs[-1].data shape:', im_info.shape)
im_info = im_info[0]
# print('im_info shape:', im_info.shape)
im_scale = im_info[2]
im_h = im_info[0]
im_w = im_info[1]
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(train)
blobs = {k: [] for k in output_blob_names}
hg_rois = rois * 1. / im_scale * np.array(
[1, 384.0 / im_w, 384.0 / im_h, 384.0 / im_w, 384.0 / im_h],
dtype=np.float32)
# hg_rois = rois[:, 1:5] * 1. / im_scale * np.array([1, 255.0/im_w, 255.0/im_h, 255.0/im_w, 255.0/im_h], dtype=np.float32)
blobs['rois'] = rois
blobs['rois_hg'] = hg_rois
fpn.add_multilevel_roi_blobs(blobs, 'rois', blobs['rois'], lvls, lvl_min,
lvl_max)
fpn.add_multilevel_roi_blobs(blobs, 'rois_hg', blobs['rois_hg'], lvls,
lvl_min, lvl_max)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_info, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
# output[0]存储所有的rois
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
# 对fpn的每一层创建新的rois
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
# 选取本层的roi索引
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
# rois中每一项在重新分配到fpn后的连接数组中的位置
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def forward(self, inputs, outputs):
"""See modeling.detector.DistributeCascadeProposals for
inputs/outputs documentation.
"""
rois = inputs[0].data
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = cascade_rcnn_roi_data.get_cascade_rcnn_blob_names(
self._stage)
blobs = {k: [] for k in output_blob_names}
# 进行rois映射到了合适的fpn层， 并重新进行采样构成训练数据
cascade_rcnn_roi_data.add_cascade_rcnn_blobs(
blobs, im_scales, roidb, self._stage)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
roi_data.cascade_rcnn.get_cascade_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
###对每一个lvl构建新的输出blob
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
###获取属于该lvl的rois的索引（行数）
idx_lvl = np.where(lvls == lvl)[0]
###将属于该lvl的所有rois放进新的blob中
blob_roi_level = rois[idx_lvl, :]
###将该lvl的blob放入最终的输出blob对应位置中
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
###这里又是在干嘛没看懂
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateTrackingLabels for inputs/outputs
documentation.
"""
track_ids_one = inputs[0].data
track_ids_two = inputs[1].data
tracking_labels = np.array([id_one == id_two \
for id_one in track_ids_one for id_two in track_ids_two], dtype=np.int32)
n_matches = sum(tracking_labels)
assert n_matches > 0, "Image pair with no matches encountered"
assert len(tracking_labels) - n_matches > 0, "Image pair with only matches encountered"
blob_utils.py_op_copy_blob(tracking_labels, outputs[0])
def forward(self, inputs, outputs):
# The inputs contains [bbox_pred, cls_prob, rois]
# rois --> np.array((num,5)), (batch_idx, x1, y2, x2, y2)
# print('++++++++++++++++++++++ Decode BBox of rcnn stage {} +++++++++++++++++++++++'.format(self._stage_num))
cls_prob = inputs[0].data[...]
bbox_pred = inputs[1].data[...]
rois = inputs[2].data[...]
if self._train:
overlaps = inputs[3].data[...]
im_info = inputs[4].data
else:
im_info = inputs[3].data
proposals_next = rois[:, 1:5]
# Use delta with max cls_score as deltas adding to rois
if cfg.MODEL.CLS_AGNOSTIC_BBOX_REG:
delta = bbox_pred[:, 4:bbox_pred.shape[1]]
else:
cls_idx = cls_prob.argmax(axis=1)
delta = np.zeros((bbox_pred.shape[0], 4), dtype=bbox_pred.dtype)
for i in range(cls_idx.shape[0]):
delta[i, :] = bbox_pred[i, cls_idx[i] * 4:cls_idx[i] * 4 + 4]
# Add bbox deltas onto rois to generate new rois
if self._stage_num == 1:
bbox_reg_weights = cfg.CASCADERCNN.BBOX_REG_WEIGHTS_STAGE1
elif self._stage_num == 2:
bbox_reg_weights = cfg.CASCADERCNN.BBOX_REG_WEIGHTS_STAGE2
new_rois = box_utils.bbox_transform(proposals_next, delta,
bbox_reg_weights)
batch_idxs = rois[:, 0].reshape(rois.shape[0], 1)
new_rois = np.hstack((batch_idxs, new_rois))
# remove invalid boxes
output_rois = remove_invalid_boxes(new_rois)
if self._train:
# screen out high IOU boxes, to remove redundant gt boxes
output_rois = remove_high_iou_boxes(output_rois, overlaps)
else:
output_rois = output_rois
# clip tiled boxes into image
output_rois = clip_tiled_bboxes(output_rois, im_info[0, :2])
blob_utils.py_op_copy_blob(output_rois, outputs[0])
def forward(self, inputs, outputs):
rois, transfer_rois = collect(inputs, self._train, self._mc)
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb,
transfer_rois)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.DistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is [rois] out from decode_bbox operator
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
_rois = inputs[0].data
rois = remove_invalid_boxes(_rois, self._stage_num)
# print('++++++++++++++++ DFRP Op of RCNN stage {} ++++++++++++++++++'.format(self._stage_num))
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[2].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[1].data)
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_cascade_fast_rcnn_blob_names(
is_training=True, stage_num=self._stage_num)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb,
self._stage_num)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
# reset roidb for next rcnn stage, remove 'max_overlaps', 'max_classes', 'bbox_targets' in each roidb,
# intialize 'boxes', 'seg_areas', 'gt_classes', 'gt_overlaps', 'box_to_gt_ind_map' only contain gt infos
# if self._stage_num == 2:
# json_dataset.reset_roidb_for_next_stage(roidb)
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train, self._stage_num)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def train_model():
"""Model training loop."""
logger = logging.getLogger(__name__)
model, weights_file, start_iter, checkpoints, output_dir = create_model(
) #for create model
if 'final' in checkpoints:
# The final model was found in the output directory, so nothing to do
return checkpoints
if 0:
output_dir = '/home/icubic/daily_work/code/Detectron/train/coco_2014_train_ET_PH_part/generalized_rcnn_multi/'
#output_dir = output_dir + '_101'
setup_model_for_training(model, weights_file, output_dir)
training_stats = TrainingStats(model)
uuuu = model.roi_data_loader._blobs_queue_name
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
print('------------train.py')
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
#aaa_debug = workspace.FetchBlob('gpu_0/data')
#bbb_debug = workspace.FetchBlob('gpu_0/conv1_w')
#ccc_debug = workspace.FetchBlob('gpu_0/'+uuuu)
try:
workspace.RunNet(model.net.Proto().name)
if 0:
#import detectron.utils.blob as blob_utils
inputs = [workspace.FetchBlob("gpu_0/rpn_rois_fpn2"),workspace.FetchBlob("gpu_0/rpn_rois_fpn3"),workspace.FetchBlob("gpu_0/rpn_rois_fpn4"),workspace.FetchBlob("gpu_0/rpn_rois_fpn5"), \
workspace.FetchBlob("gpu_0/rpn_rois_fpn6"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn2"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn3"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn4"), \
workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn5"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn6"),workspace.FetchBlob("gpu_0/roidb"),workspace.FetchBlob("gpu_0/im_info"),\
]
rois = collect(inputs, True)
#inputs.append(workspace.FetchBlob("gpu_0/rpn_rois_fpn2"))
im_info = inputs[-1]
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2])
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb,
rois,
im_scales,
crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(
)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
#if (np.sum(bb == 1))>0:
# print('cc')
except:
aa = workspace.FetchBlob("gpu_0/rpn_rois_fpn2")
aaa_debug = workspace.FetchBlob('gpu_0/data')
print('aaaaaerror')
#print("blobs:\n{}".format(workspace.Blobs()))
#print('train.py aaaaaaaa_debug')
if 1:
aaa = workspace.FetchBlob("gpu_0/data") # nchw
#img = aaa[1].copy()
# BGR HWC -> CHW 12
#transform_img = img.swapaxes(0, 1).swapaxes(1, 2)
#cv2.imshow("image0 ", transform_img[:, :, (2, 1, 0)])
#cv2.waitKey(0)
#cv2.destroyAllWindows()
#cv2.imshow('/home/icubic/daily_work/code/Detectron/aaa.png', aaa[0])
aaa_debug = workspace.FetchBlob('gpu_0/data')
bbb_debug = workspace.FetchBlob('gpu_0/conv1_w')
ccc_debug = workspace.FetchBlob('gpu_0/' + uuuu)
ddd_debug = workspace.FetchBlob('gpu_0/roidb')
eee_debug = workspace.FetchBlob('gpu_0/im_info')
#print("Fetched data:\n{}".format(workspace.FetchBlob("gpu_0/data")))
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats()
training_stats.LogIterStats(cur_iter, lr)
if (cur_iter + 1) % (
CHECKPOINT_PERIOD / 4
) == 0 and cur_iter > start_iter: #((cur_iter + 1) % (CHECKPOINT_PERIOD/1) == 0 and (cur_iter > start_iter and cur_iter < 50000)) or ((cur_iter + 1) % (CHECKPOINT_PERIOD/8) == 0 and cur_iter > 50000):
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter_50_{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if cur_iter == start_iter + training_stats.LOG_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
logger.critical('Loss is NaN, exiting...')
model.roi_data_loader.shutdown()
envu.exit_on_error()
# Save the final model
checkpoints['final'] = os.path.join(output_dir, 'model_final_50.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# Shutdown data loading threads
model.roi_data_loader.shutdown()
return checkpoints
