def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_info, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
if cfg.POLYGON.POLYGON_ON:
output_blob_names = polygon_rcnn_roi_data.get_polygon_rcnn_blob_names(
)
else:
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
if cfg.POLYGON.POLYGON_ON:
roidb_utils.add_polygon_regression_targets(roidb)
else:
roidb_utils.add_bbox_regression_targets(roidb)
blobs = {k: [] for k in output_blob_names}
if cfg.POLYGON.POLYGON_ON:
polygon_rcnn_roi_data.add_polygon_rcnn_blobs(
blobs, im_scales, roidb)
else:
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def get_minibatch_s6(roidb, roidb_noclass):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
if 0:
random_bbox = dict()
random_bbox['kernel_size'] = 224
random_bbox['tl_x'] = random.randint(0, 800)
random_bbox['tl_y'] = random.randint(0, 800)
blobs = {k: [] for k in get_minibatch_blob_names()}
# Get the input image blob, formatted for caffe2
im_blob, im_scales, error_flag = _get_image_blob_s6(roidb, roidb_noclass)
blobs['data'] = im_blob
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb,
im_width, im_height)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs, valid
def get_minibatch_mul(roidb):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
blobs = {k: [] for k in get_minibatch_blob_names()}
#print("minibatch.py 157 ---------{}".format(len(roidb)))
# Get the input image blob, formatted for caffe2
im_blob, im_scales, roidb_ = _get_img_blob_mul(roidb)
blobs['data'] = im_blob
if cfg.RPN.RPN_ON:
# # RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb_)
return blobs, valid
if cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales,
roidb_, im_width,
im_height)
else:
# Fast R-CNN like models trained on
# precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales,
roidb_)
return blobs, valid
def get_minibatch(roidb):
"""载入roidb,构建一个minibatch采样."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
# 生成空字典数据 如 {'data': [], 'fpn_labels_int32_wide_fpn1': []}等
#print("现在执行的是无扩增版本")
blobs = {k: [] for k in get_minibatch_blob_names()}
print("blobs is ", blobs)
# Get the input image blob, formatted for caffe2
im_blob, im_scales = _get_image_blob(roidb)
blobs['data'] = im_blob
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb,
im_width, im_height)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs, valid
def get_minibatch(roidb):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
blobs = {k: [] for k in get_minibatch_blob_names()}
# Get the input image blob, formatted for caffe2
im_blob, im_scales = _get_image_blob(roidb)
blobs['data'] = im_blob
# Calculate tracking metadata if using pre-calculated blobs
if len(cfg.DATA_LOADER.EXTRA_BLOBS):
blobs_extra = _get_extra_blobs(roidb)
blobs.update(blobs_extra)
if 'track_n_rois' in cfg.DATA_LOADER.EXTRA_BLOBS:
blobs.update({
'track_n_rois_one': [blobs_extra['track_n_rois'][0]],
'track_n_rois_two': [blobs_extra['track_n_rois'][1]],
})
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb,
im_width, im_height)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs, valid
def get_minibatch(roidb):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
blobs = {k: [] for k in get_minibatch_blob_names()}
# Get the input image blob, formatted for caffe2
im_blob, im_scales, im_crops = _get_image_blob(roidb)
blobs['data'] = im_blob
# row col row col to x1 y1 x2 y2
im_crops = np.array(im_crops, dtype=np.int32)
im_crops = im_crops[:, (1, 0, 3, 2)]
if cfg.REID.APM:
valid = reid_apm_roi_data.add_reid_blobs(blobs, im_scales, im_crops,
roidb)
else:
valid = reid_roi_data.add_reid_blobs(blobs, im_scales, roidb)
return blobs, valid
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(blobs, im_scales, roidb,
im_width, im_height)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs, valid
def forward(self, inputs, outputs):
rois, transfer_rois = collect(inputs, self._train, self._mc)
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb,
transfer_rois)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.DistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is [rois] out from decode_bbox operator
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
_rois = inputs[0].data
rois = remove_invalid_boxes(_rois, self._stage_num)
# print('++++++++++++++++ DFRP Op of RCNN stage {} ++++++++++++++++++'.format(self._stage_num))
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[2].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[1].data)
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_cascade_fast_rcnn_blob_names(
is_training=True, stage_num=self._stage_num)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb,
self._stage_num)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
# reset roidb for next rcnn stage, remove 'max_overlaps', 'max_classes', 'bbox_targets' in each roidb,
# intialize 'boxes', 'seg_areas', 'gt_classes', 'gt_overlaps', 'box_to_gt_ind_map' only contain gt infos
# if self._stage_num == 2:
# json_dataset.reset_roidb_for_next_stage(roidb)
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train, self._stage_num)
def get_minibatch(roidb):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
blobs = {k: [] for k in get_minibatch_blob_names()}
# Get the input image blob, formatted for caffe2
im_blob, im_scales = _get_image_blob(roidb)
blobs['data'] = im_blob
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(
blobs, im_scales, roidb, im_width, im_height
)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs, valid
def get_minibatch(roidb):
"""Given a roidb, construct a minibatch sampled from it."""
# We collect blobs from each image onto a list and then concat them into a
# single tensor, hence we initialize each blob to an empty list
blobs = {k: [] for k in get_minibatch_blob_names()}
# Get the input image blob, formatted for caffe2
# im_blob, im_scales = _get_image_blob(roidb)
im_blob, im_scales, pose_pred, pose_line, blobs['seg_gt_label'] = _get_image_pose_blob(roidb) # pose_pred the same shape with im_blob
blobs['data'] = im_blob
blobs['normalizer'] = np.array([100], dtype=np.float32)
if 'LIP' in cfg.TRAIN.DATASETS[0]:
blobs['pose_pred_4'], blobs['pose_pred_8'], blobs['pose_pred_16'], blobs['pose_pred_32'] = _resize_pose_blob(pose_pred, channel=26)
else:
blobs['pose_pred_4'], blobs['pose_pred_8'], blobs['pose_pred_16'], blobs['pose_pred_32'] = _resize_pose_blob(pose_pred, channel=26)
# blobs['pose_pred_8'], blobs['pose_pred_16'] = _resize_pose_blob_to13(pose_pred) # pose 16 to 13 channel
# blobs['pose_sum_8'], blobs['pose_sum_16'] = pose_sum_to_onehotmap(blobs['pose_pred_8'], blobs['pose_pred_16'])
blobs['pose_line_8'], blobs['pose_line_16'] = _resize_poseline_blob(pose_line)
if cfg.RPN.RPN_ON:
# RPN-only or end-to-end Faster/Mask R-CNN
valid = rpn_roi_data.add_rpn_blobs(blobs, im_scales, roidb)
elif cfg.RETINANET.RETINANET_ON:
im_width, im_height = im_blob.shape[3], im_blob.shape[2]
# im_width, im_height corresponds to the network input: padded image
# (if needed) width and height. We pass it as input and slice the data
# accordingly so that we don't need to use SampleAsOp
valid = retinanet_roi_data.add_retinanet_blobs(
blobs, im_scales, roidb, im_width, im_height
)
else:
# Fast R-CNN like models trained on precomputed proposals
valid = fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
# blobs['pose_pred'] = pose_pred_model.pred_pose_batch(roidb)
# pose_pred_model.draw_batch(blobs['pose_pred'], roidb)
# blobs['pose_pred'] = _get_pose_pred(roidb)
# logger.info(blobs['pose_pred'].shape)
return blobs, valid
def train_model():
"""Model training loop."""
logger = logging.getLogger(__name__)
model, weights_file, start_iter, checkpoints, output_dir = create_model(
) #for create model
if 'final' in checkpoints:
# The final model was found in the output directory, so nothing to do
return checkpoints
if 0:
output_dir = '/home/icubic/daily_work/code/Detectron/train/coco_2014_train_ET_PH_part/generalized_rcnn_multi/'
#output_dir = output_dir + '_101'
setup_model_for_training(model, weights_file, output_dir)
training_stats = TrainingStats(model)
uuuu = model.roi_data_loader._blobs_queue_name
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
print('------------train.py')
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
#aaa_debug = workspace.FetchBlob('gpu_0/data')
#bbb_debug = workspace.FetchBlob('gpu_0/conv1_w')
#ccc_debug = workspace.FetchBlob('gpu_0/'+uuuu)
try:
workspace.RunNet(model.net.Proto().name)
if 0:
#import detectron.utils.blob as blob_utils
inputs = [workspace.FetchBlob("gpu_0/rpn_rois_fpn2"),workspace.FetchBlob("gpu_0/rpn_rois_fpn3"),workspace.FetchBlob("gpu_0/rpn_rois_fpn4"),workspace.FetchBlob("gpu_0/rpn_rois_fpn5"), \
workspace.FetchBlob("gpu_0/rpn_rois_fpn6"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn2"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn3"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn4"), \
workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn5"),workspace.FetchBlob("gpu_0/rpn_roi_probs_fpn6"),workspace.FetchBlob("gpu_0/roidb"),workspace.FetchBlob("gpu_0/im_info"),\
]
rois = collect(inputs, True)
#inputs.append(workspace.FetchBlob("gpu_0/rpn_rois_fpn2"))
im_info = inputs[-1]
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2])
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb,
rois,
im_scales,
crowd_thresh=0)
roidb_utils.add_bbox_regression_targets(roidb)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(
)
blobs = {k: [] for k in output_blob_names}
fast_rcnn_roi_data.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
#if (np.sum(bb == 1))>0:
# print('cc')
except:
aa = workspace.FetchBlob("gpu_0/rpn_rois_fpn2")
aaa_debug = workspace.FetchBlob('gpu_0/data')
print('aaaaaerror')
#print("blobs:\n{}".format(workspace.Blobs()))
#print('train.py aaaaaaaa_debug')
if 1:
aaa = workspace.FetchBlob("gpu_0/data") # nchw
#img = aaa[1].copy()
# BGR HWC -> CHW 12
#transform_img = img.swapaxes(0, 1).swapaxes(1, 2)
#cv2.imshow("image0 ", transform_img[:, :, (2, 1, 0)])
#cv2.waitKey(0)
#cv2.destroyAllWindows()
#cv2.imshow('/home/icubic/daily_work/code/Detectron/aaa.png', aaa[0])
aaa_debug = workspace.FetchBlob('gpu_0/data')
bbb_debug = workspace.FetchBlob('gpu_0/conv1_w')
ccc_debug = workspace.FetchBlob('gpu_0/' + uuuu)
ddd_debug = workspace.FetchBlob('gpu_0/roidb')
eee_debug = workspace.FetchBlob('gpu_0/im_info')
#print("Fetched data:\n{}".format(workspace.FetchBlob("gpu_0/data")))
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats()
training_stats.LogIterStats(cur_iter, lr)
if (cur_iter + 1) % (
CHECKPOINT_PERIOD / 4
) == 0 and cur_iter > start_iter: #((cur_iter + 1) % (CHECKPOINT_PERIOD/1) == 0 and (cur_iter > start_iter and cur_iter < 50000)) or ((cur_iter + 1) % (CHECKPOINT_PERIOD/8) == 0 and cur_iter > 50000):
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter_50_{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if cur_iter == start_iter + training_stats.LOG_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
logger.critical('Loss is NaN, exiting...')
model.roi_data_loader.shutdown()
envu.exit_on_error()
# Save the final model
checkpoints['final'] = os.path.join(output_dir, 'model_final_50.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# Shutdown data loading threads
model.roi_data_loader.shutdown()
return checkpoints
