def __init__(self, configer):
self.configer = configer
self.det_visualizer = DetVisualizer(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.default_boxes = PriorBoxLayer(configer)()
self.device = torch.device('cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.yolo_target_generator = YOLOTargetGenerator(configer)
self.yolo_detection_layer = YOLODetectionLayer(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.ssd_priorbox_layer = SSDPriorBoxLayer(configer)
self.ssd_target_generator = SSDTargetGenerator(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.roi_sampler = FRROISampler(configer)
self.rpn_target_generator = RPNTargetAssigner(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.fr_roi_generator = FRROIGenerator(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = LossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.det_running_score = DetRunningScore(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self.runner_state = dict()
self._init_model()
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = LossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.yolo_detection_layer = YOLODetectionLayer(configer)
self.yolo_target_generator = YOLOTargetGenerator(configer)
self.det_running_score = DetRunningScore(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self.runner_state = dict()
self._init_model()
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.rpn_target_generator = RPNTargetGenerator(configer)
self.det_running_score = DetRunningScore(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
class YOLOv3(object):
"""
The class for YOLO v3. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = LossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.yolo_detection_layer = YOLODetectionLayer(configer)
self.yolo_target_generator = YOLOTargetGenerator(configer)
self.det_running_score = DetRunningScore(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self.runner_state = dict()
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = RunnerHelper.load_net(self, self.det_net)
self.optimizer, self.scheduler = Trainer.init(self,
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
self.det_loss = self.det_loss_manager.get_det_loss()
def _get_parameters(self):
lr_1 = []
lr_10 = []
params_dict = dict(self.det_net.named_parameters())
for key, value in params_dict.items():
if 'backbone' not in key:
lr_10.append(value)
else:
lr_1.append(value)
params = [{
'params': lr_1,
'lr': self.configer.get('lr', 'base_lr')
}, {
'params': lr_10,
'lr': self.configer.get('lr', 'base_lr') * 10.
}]
return params
def train(self):
"""
Train function of every epoch during train phase.
"""
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.runner_state['epoch'] += 1
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_dict in enumerate(self.train_loader):
Trainer.update(self, backbone_list=(0, ))
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs = RunnerHelper.to_device(self, inputs)
# Forward pass.
feat_list, predictions, _ = self.det_net(inputs)
targets, objmask, noobjmask = self.yolo_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels, input_size)
targets, objmask, noobjmask = RunnerHelper.to_device(
self, targets, objmask, noobjmask)
# Compute the loss of the train batch & backward.
loss = self.det_loss(predictions, targets, objmask, noobjmask)
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.runner_state['iters'] += 1
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.runner_state['epoch'],
self.runner_state['iters'],
self.configer.get('solver', 'display_iter'),
RunnerHelper.get_lr(self.optimizer),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
if self.configer.get('lr', 'metric') == 'iters' \
and self.runner_state['iters'] == self.configer.get('solver', 'max_iters'):
break
# Check to val the current model.
if self.runner_state['iters'] % self.configer.get(
'solver', 'test_interval') == 0:
self.val()
def val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for i, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
# Forward pass.
inputs = RunnerHelper.to_device(self, inputs)
feat_list, predictions, detections = self.det_net(inputs)
targets, objmask, noobjmask = self.yolo_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels, input_size)
targets, objmask, noobjmask = RunnerHelper.to_device(
self, targets, objmask, noobjmask)
# Compute the loss of the val batch.
loss = self.det_loss(predictions, targets, objmask, noobjmask)
self.val_losses.update(loss.item(), inputs.size(0))
batch_detections = YOLOv3Test.decode(detections, self.configer,
input_size)
batch_pred_bboxes = self.__get_object_list(
batch_detections, input_size)
self.det_running_score.update(batch_pred_bboxes,
batch_gt_bboxes, batch_gt_labels)
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
RunnerHelper.save_net(self,
self.det_net,
iters=self.runner_state['iters'])
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
Log.info('Val mAP: {}'.format(self.det_running_score.get_mAP()))
self.det_running_score.reset()
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def __get_object_list(self, batch_detections, input_size):
batch_pred_bboxes = list()
for idx, detections in enumerate(batch_detections):
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
xmin = x1.cpu().item()
ymin = y1.cpu().item()
xmax = x2.cpu().item()
ymax = y2.cpu().item()
cf = conf.cpu().item()
cls_pred = cls_pred.cpu().item()
object_list.append([
xmin, ymin, xmax, ymax,
int(cls_pred),
float('%.2f' % cf)
])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
class FastRCNNTest(object):
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.roi_sampler = FRRoiSampleLayer(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.rpn_target_generator = RPNTargetGenerator(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.fr_roi_generator = FRRoiGenerator(configer)
self.data_transformer = DataTransformer(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.det_net.eval()
def __test_img(self, image_path, json_path, raw_path, vis_path):
Log.info('Image Path: {}'.format(image_path))
img = ImageHelper.read_image(
image_path,
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
ori_img_bgr = ImageHelper.get_cv2_bgr(img,
mode=self.configer.get(
'data', 'input_mode'))
img, scale = BoundResize()(img)
inputs = self.blob_helper.make_input(img, scale=1.0)
with torch.no_grad():
# Forward pass.
test_group = self.det_net(inputs, scale)
test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group
batch_detections = self.decode(test_roi_locs, test_roi_scores,
test_indices_and_rois,
test_rois_num, self.configer,
ImageHelper.get_size(img))
json_dict = self.__get_info_tree(batch_detections[0],
ori_img_bgr,
scale=scale)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('vis', 'conf_threshold'))
cv2.imwrite(vis_path, image_canvas)
cv2.imwrite(raw_path, ori_img_bgr)
Log.info('Json Path: {}'.format(json_path))
JsonHelper.save_file(json_dict, json_path)
return json_dict
@staticmethod
def decode(roi_locs, roi_scores, indices_and_rois, test_rois_num, configer,
input_size):
roi_locs = roi_locs.cpu()
roi_scores = roi_scores.cpu()
indices_and_rois = indices_and_rois.cpu()
num_classes = configer.get('data', 'num_classes')
mean = torch.Tensor(configer.get(
'roi', 'loc_normalize_mean')).repeat(num_classes)[None]
std = torch.Tensor(configer.get(
'roi', 'loc_normalize_std')).repeat(num_classes)[None]
mean = mean.to(roi_locs.device)
std = std.to(roi_locs.device)
roi_locs = (roi_locs * std + mean)
roi_locs = roi_locs.contiguous().view(-1, num_classes, 4)
# roi_locs = roi_locs[:,:, [1, 0, 3, 2]]
rois = indices_and_rois[:, 1:]
rois = rois.contiguous().view(-1, 1, 4).expand_as(roi_locs)
wh = torch.exp(roi_locs[:, :, 2:]) * (rois[:, :, 2:] - rois[:, :, :2])
cxcy = roi_locs[:, :, :2] * (rois[:, :, 2:] - rois[:, :, :2]) + (
rois[:, :, :2] + rois[:, :, 2:]) / 2
dst_bbox = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 2) # [b, 8732,4]
# clip bounding box
dst_bbox[:, :, 0::2] = (dst_bbox[:, :,
0::2]).clamp(min=0,
max=input_size[0] - 1)
dst_bbox[:, :, 1::2] = (dst_bbox[:, :,
1::2]).clamp(min=0,
max=input_size[1] - 1)
if configer.get('phase') != 'debug':
cls_prob = F.softmax(roi_scores, dim=1)
else:
cls_prob = roi_scores
cls_label = torch.LongTensor([i for i in range(num_classes)])\
.contiguous().view(1, num_classes).repeat(indices_and_rois.size(0), 1)
output = [None for _ in range(test_rois_num.size(0))]
start_index = 0
for i in range(test_rois_num.size(0)):
# batch_index = (indices_and_rois[:, 0] == i).nonzero().contiguous().view(-1,)
# tmp_dst_bbox = dst_bbox[batch_index]
# tmp_cls_prob = cls_prob[batch_index]
# tmp_cls_label = cls_label[batch_index]
tmp_dst_bbox = dst_bbox[start_index:start_index + test_rois_num[i]]
tmp_cls_prob = cls_prob[start_index:start_index + test_rois_num[i]]
tmp_cls_label = cls_label[start_index:start_index +
test_rois_num[i]]
start_index += test_rois_num[i]
mask = (tmp_cls_prob > configer.get(
'vis', 'conf_threshold')) & (tmp_cls_label > 0)
tmp_dst_bbox = tmp_dst_bbox[mask].contiguous().view(-1, 4)
if tmp_dst_bbox.numel() == 0:
continue
tmp_cls_prob = tmp_cls_prob[mask].contiguous().view(
-1, ).unsqueeze(1)
tmp_cls_label = tmp_cls_label[mask].contiguous().view(
-1, ).unsqueeze(1)
valid_preds = torch.cat(
(tmp_dst_bbox, tmp_cls_prob.float(), tmp_cls_label.float()), 1)
keep = DetHelper.cls_nms(valid_preds[:, :4],
scores=valid_preds[:, 4],
labels=valid_preds[:, 5],
nms_threshold=configer.get(
'nms', 'overlap_threshold'),
iou_mode=configer.get('nms', 'mode'))
output[i] = valid_preds[keep]
return output
def __make_tensor(self, gt_bboxes, gt_labels):
len_arr = [gt_labels[i].numel() for i in range(len(gt_bboxes))]
batch_maxlen = max(max(len_arr), 1)
target_bboxes = torch.zeros((len(gt_bboxes), batch_maxlen, 4)).float()
target_labels = torch.zeros((len(gt_bboxes), batch_maxlen)).long()
for i in range(len(gt_bboxes)):
target_bboxes[i, :len_arr[i], :] = gt_bboxes[i]
target_labels[i, :len_arr[i]] = gt_labels[i]
target_bboxes_num = torch.Tensor(len_arr).long()
return target_bboxes, target_bboxes_num, target_labels
def __get_info_tree(self, detections, image_raw, scale=1.0):
height, width, _ = image_raw.shape
json_dict = dict()
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
object_dict = dict()
xmin = min(x1.cpu().item() / scale, width - 1)
ymin = min(y1.cpu().item() / scale, height - 1)
xmax = min(x2.cpu().item() / scale, width - 1)
ymax = min(y2.cpu().item() / scale, height - 1)
object_dict['bbox'] = [xmin, ymin, xmax, ymax]
object_dict['label'] = int(cls_pred.cpu().item()) - 1
object_dict['score'] = float('%.2f' % conf.cpu().item())
object_list.append(object_dict)
json_dict['objects'] = object_list
return json_dict
def test(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'val/results/det',
self.configer.get('dataset'))
test_img = self.configer.get('test_img')
test_dir = self.configer.get('test_dir')
if test_img is None and test_dir is None:
Log.error('test_img & test_dir not exists.')
exit(1)
if test_img is not None and test_dir is not None:
Log.error('Either test_img or test_dir.')
exit(1)
if test_img is not None:
base_dir = os.path.join(base_dir, 'test_img')
filename = test_img.rstrip().split('/')[-1]
json_path = os.path.join(
base_dir, 'json',
'{}.json'.format('.'.join(filename.split('.')[:-1])))
raw_path = os.path.join(base_dir, 'raw', filename)
vis_path = os.path.join(
base_dir, 'vis',
'{}_vis.png'.format('.'.join(filename.split('.')[:-1])))
FileHelper.make_dirs(json_path, is_file=True)
FileHelper.make_dirs(raw_path, is_file=True)
FileHelper.make_dirs(vis_path, is_file=True)
self.__test_img(test_img, json_path, raw_path, vis_path)
else:
base_dir = os.path.join(base_dir, 'test_dir',
test_dir.rstrip('/').split('/')[-1])
FileHelper.make_dirs(base_dir)
for filename in FileHelper.list_dir(test_dir):
image_path = os.path.join(test_dir, filename)
json_path = os.path.join(
base_dir, 'json',
'{}.json'.format('.'.join(filename.split('.')[:-1])))
raw_path = os.path.join(base_dir, 'raw', filename)
vis_path = os.path.join(
base_dir, 'vis',
'{}_vis.png'.format('.'.join(filename.split('.')[:-1])))
FileHelper.make_dirs(json_path, is_file=True)
FileHelper.make_dirs(raw_path, is_file=True)
FileHelper.make_dirs(vis_path, is_file=True)
self.__test_img(image_path, json_path, raw_path, vis_path)
def debug(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'vis/results/det',
self.configer.get('dataset'), 'debug')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
count = 0
for i, data_dict in enumerate(self.det_data_loader.get_trainloader()):
img_scale = data_dict['imgscale']
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
# batch_gt_bboxes = ResizeBoxes()(inputs, data_dict['bboxes'])
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
feat_list = list()
for stride in self.configer.get('rpn', 'stride_list'):
feat_list.append(
torch.zeros((inputs.size(0), 1, input_size[1] // stride,
input_size[0] // stride)))
gt_rpn_locs, gt_rpn_labels = self.rpn_target_generator(
feat_list, batch_gt_bboxes, input_size)
eye_matrix = torch.eye(2)
gt_rpn_labels[gt_rpn_labels == -1] = 0
gt_rpn_scores = eye_matrix[gt_rpn_labels.view(-1)].view(
inputs.size(0), -1, 2)
test_indices_and_rois, _ = self.fr_roi_generator(
feat_list, gt_rpn_locs, gt_rpn_scores,
self.configer.get('rpn', 'n_test_pre_nms'),
self.configer.get('rpn', 'n_test_post_nms'), input_size,
img_scale)
gt_bboxes, gt_nums, gt_labels = self.__make_tensor(
batch_gt_bboxes, batch_gt_labels)
sample_rois, gt_roi_locs, gt_roi_labels = self.roi_sampler(
test_indices_and_rois, gt_bboxes, gt_nums, gt_labels,
input_size)
self.det_visualizer.vis_rois(inputs,
sample_rois[gt_roi_labels > 0])
gt_cls_roi_locs = torch.zeros(
(gt_roi_locs.size(0), self.configer.get('data',
'num_classes'), 4))
gt_cls_roi_locs[torch.arange(0, sample_rois.size(0)).long(),
gt_roi_labels.long()] = gt_roi_locs
gt_cls_roi_locs = gt_cls_roi_locs.contiguous().view(
-1, 4 * self.configer.get('data', 'num_classes'))
eye_matrix = torch.eye(self.configer.get('data', 'num_classes'))
gt_roi_scores = eye_matrix[gt_roi_labels.view(-1)].view(
gt_roi_labels.size(0),
self.configer.get('data', 'num_classes'))
test_rois_num = torch.zeros((len(gt_bboxes), )).long()
for batch_id in range(len(gt_bboxes)):
batch_index = (
sample_rois[:, 0] == batch_id).nonzero().contiguous().view(
-1, )
test_rois_num[batch_id] = batch_index.numel()
batch_detections = FastRCNNTest.decode(gt_cls_roi_locs,
gt_roi_scores, sample_rois,
test_rois_num,
self.configer, input_size)
for j in range(inputs.size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img_bgr = self.blob_helper.tensor2bgr(inputs[j])
self.det_visualizer.vis_default_bboxes(
ori_img_bgr, self.fr_priorbox_layer(feat_list, input_size),
gt_rpn_labels[j])
json_dict = self.__get_info_tree(batch_detections[j],
ori_img_bgr)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('vis', 'conf_threshold'))
cv2.imwrite(
os.path.join(base_dir, '{}_{}_vis.png'.format(i, j)),
image_canvas)
cv2.imshow('main', image_canvas)
cv2.waitKey()
class FasterRCNN(object):
"""
The class for Single Shot Detector. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.rpn_target_generator = RPNTargetGenerator(configer)
self.det_running_score = DetRunningScore(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
self.fr_loss = self.det_loss_manager.get_det_loss('fr_loss')
def _get_parameters(self):
lr_1 = []
lr_2 = []
params_dict = dict(self.det_net.named_parameters())
for key, value in params_dict.items():
if value.requires_grad:
if 'bias' in key:
lr_2.append(value)
else:
lr_1.append(value)
params = [{
'params': lr_1,
'lr': self.configer.get('lr', 'base_lr')
}, {
'params': lr_2,
'lr': self.configer.get('lr', 'base_lr') * 2.,
'weight_decay': 0
}]
return params
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
for i, data_dict in enumerate(self.train_loader):
inputs = data_dict['img']
img_scale = data_dict['imgscale']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
self.data_time.update(time.time() - start_time)
# Change the data type.
gt_bboxes, gt_nums, gt_labels = self.__make_tensor(
batch_gt_bboxes, batch_gt_labels)
gt_bboxes, gt_num, gt_labels = self.module_utilizer.to_device(
gt_bboxes, gt_nums, gt_labels)
inputs = self.module_utilizer.to_device(inputs)
# Forward pass.
feat_list, train_group = self.det_net(inputs, gt_bboxes, gt_num,
gt_labels, img_scale)
gt_rpn_locs, gt_rpn_labels = self.rpn_target_generator(
feat_list, batch_gt_bboxes,
[inputs.size(3), inputs.size(2)])
gt_rpn_locs, gt_rpn_labels = self.module_utilizer.to_device(
gt_rpn_locs, gt_rpn_labels)
rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores, gt_roi_bboxes, gt_roi_labels = train_group
# Compute the loss of the train batch & backward.
loss = self.fr_loss(
[rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores],
[gt_rpn_locs, gt_rpn_labels, gt_roi_bboxes, gt_roi_labels])
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.module_utilizer.clip_grad(self.det_net, 10.)
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
(self.configer.get('iters')) % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for j, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
img_scale = data_dict['imgscale']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
# Change the data type.
gt_bboxes, gt_nums, gt_labels = self.__make_tensor(
batch_gt_bboxes, batch_gt_labels)
gt_bboxes, gt_num, gt_labels = self.module_utilizer.to_device(
gt_bboxes, gt_nums, gt_labels)
inputs = self.module_utilizer.to_device(inputs)
# Forward pass.
feat_list, train_group, test_group = self.det_net(
inputs, gt_bboxes, gt_nums, gt_labels, img_scale)
rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores, gt_roi_bboxes, gt_roi_labels = train_group
gt_rpn_locs, gt_rpn_labels = self.rpn_target_generator(
feat_list, batch_gt_bboxes,
[inputs.size(3), inputs.size(2)])
gt_rpn_locs, gt_rpn_labels = self.module_utilizer.to_device(
gt_rpn_locs, gt_rpn_labels)
# Compute the loss of the train batch & backward.
loss = self.fr_loss(
[rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores],
[gt_rpn_locs, gt_rpn_labels, gt_roi_bboxes, gt_roi_labels])
self.val_losses.update(loss.item(), inputs.size(0))
test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group
batch_detections = FastRCNNTest.decode(
test_roi_locs, test_roi_scores, test_indices_and_rois,
test_rois_num, self.configer,
[inputs.size(3), inputs.size(2)])
batch_pred_bboxes = self.__get_object_list(batch_detections)
self.det_running_score.update(batch_pred_bboxes,
batch_gt_bboxes, batch_gt_labels)
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.det_net, save_mode='iters')
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
Log.info('Val mAP: {}\n'.format(self.det_running_score.get_mAP()))
self.det_running_score.reset()
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def __make_tensor(self, gt_bboxes, gt_labels):
len_arr = [gt_labels[i].numel() for i in range(len(gt_bboxes))]
batch_maxlen = max(max(len_arr), 1)
target_bboxes = torch.zeros((len(gt_bboxes), batch_maxlen, 4)).float()
target_labels = torch.zeros((len(gt_bboxes), batch_maxlen)).long()
for i in range(len(gt_bboxes)):
if len_arr[i] == 0:
continue
target_bboxes[i, :len_arr[i], :] = gt_bboxes[i].clone()
target_labels[i, :len_arr[i]] = gt_labels[i].clone()
target_bboxes_num = torch.Tensor(len_arr).long()
return target_bboxes, target_bboxes_num, target_labels
def __get_object_list(self, batch_detections):
batch_pred_bboxes = list()
for idx, detections in enumerate(batch_detections):
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
xmin = x1.cpu().item()
ymin = y1.cpu().item()
xmax = x2.cpu().item()
ymax = y2.cpu().item()
cf = conf.cpu().item()
cls_pred = int(cls_pred.cpu().item()) - 1
object_list.append(
[xmin, ymin, xmax, ymax, cls_pred,
float('%.2f' % cf)])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
def train(self):
cudnn.benchmark = True
if self.configer.get('network',
'resume') is not None and self.configer.get(
'network', 'resume_val'):
self.__val()
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class SingleShotDetectorTest(object):
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.ssd_priorbox_layer = SSDPriorBoxLayer(configer)
self.ssd_target_generator = SSDTargetGenerator(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = RunnerHelper.load_net(self, self.det_net)
self.det_net.eval()
def __test_img(self, image_path, json_path, raw_path, vis_path):
Log.info('Image Path: {}'.format(image_path))
img = ImageHelper.read_image(
image_path,
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
ori_img_bgr = ImageHelper.get_cv2_bgr(img,
mode=self.configer.get(
'data', 'input_mode'))
inputs = self.blob_helper.make_input(img,
input_size=self.configer.get(
'test', 'input_size'),
scale=1.0)
with torch.no_grad():
feat_list, bbox, cls = self.det_net(inputs)
batch_detections = self.decode(
bbox, cls,
self.ssd_priorbox_layer(feat_list,
self.configer.get('test', 'input_size')),
self.configer, [inputs.size(3), inputs.size(2)])
json_dict = self.__get_info_tree(
batch_detections[0], ori_img_bgr,
[inputs.size(3), inputs.size(2)])
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('res', 'vis_conf_thre'))
cv2.imwrite(vis_path, image_canvas)
cv2.imwrite(raw_path, ori_img_bgr)
Log.info('Json Path: {}'.format(json_path))
JsonHelper.save_file(json_dict, json_path)
return json_dict
@staticmethod
def decode(bbox, conf, default_boxes, configer, input_size):
loc = bbox
if configer.get('phase') != 'debug':
conf = F.softmax(conf, dim=-1)
default_boxes = default_boxes.unsqueeze(0).repeat(loc.size(0), 1,
1).to(bbox.device)
variances = [0.1, 0.2]
wh = torch.exp(loc[:, :, 2:] * variances[1]) * default_boxes[:, :, 2:]
cxcy = loc[:, :, :2] * variances[
0] * default_boxes[:, :, 2:] + default_boxes[:, :, :2]
boxes = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 2) # [b, 8732,4]
batch_size, num_priors, _ = boxes.size()
boxes = boxes.unsqueeze(2).repeat(1, 1,
configer.get('data', 'num_classes'),
1)
boxes = boxes.contiguous().view(boxes.size(0), -1, 4)
# clip bounding box
boxes[:, :, 0::2] = boxes[:, :, 0::2].clamp(min=0,
max=input_size[0] - 1)
boxes[:, :, 1::2] = boxes[:, :, 1::2].clamp(min=0,
max=input_size[1] - 1)
labels = torch.Tensor([
i for i in range(configer.get('data', 'num_classes'))
]).to(boxes.device)
labels = labels.view(1, 1, -1,
1).repeat(batch_size, num_priors, 1,
1).contiguous().view(batch_size, -1, 1)
max_conf = conf.contiguous().view(batch_size, -1, 1)
# max_conf, labels = conf.max(2, keepdim=True) # [b, 8732,1]
predictions = torch.cat((boxes, max_conf.float(), labels.float()), 2)
output = [None for _ in range(len(predictions))]
for image_i, image_pred in enumerate(predictions):
ids = labels[image_i].squeeze(1).nonzero().contiguous().view(-1, )
if ids.numel() == 0:
continue
valid_preds = image_pred[ids]
_, order = valid_preds[:, 4].sort(0, descending=True)
order = order[:configer.get('nms', 'pre_nms')]
valid_preds = valid_preds[order]
valid_preds = valid_preds[
valid_preds[:, 4] > configer.get('res', 'val_conf_thre')]
if valid_preds.numel() == 0:
continue
valid_preds = DetHelper.cls_nms(
valid_preds[:, :6],
labels=valid_preds[:, 5],
max_threshold=configer.get('nms', 'max_threshold'),
cls_keep_num=configer.get('res', 'cls_keep_num'))
_, order = valid_preds[:, 4].sort(0, descending=True)
order = order[:configer.get('res', 'max_per_image')]
output[image_i] = valid_preds[order]
return output
def __get_info_tree(self, detections, image_raw, input_size):
height, width, _ = image_raw.shape
in_width, in_height = input_size
json_dict = dict()
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
object_dict = dict()
xmin = x1.cpu().item() / in_width * width
ymin = y1.cpu().item() / in_height * height
xmax = x2.cpu().item() / in_width * width
ymax = y2.cpu().item() / in_height * height
object_dict['bbox'] = [xmin, ymin, xmax, ymax]
object_dict['label'] = int(cls_pred.cpu().item()) - 1
object_dict['score'] = float('%.2f' % conf.cpu().item())
object_list.append(object_dict)
json_dict['objects'] = object_list
return json_dict
def debug(self, vis_dir):
count = 0
for i, data_dict in enumerate(self.det_data_loader.get_trainloader()):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
feat_list = list()
for stride in self.configer.get('network', 'stride_list'):
feat_list.append(
torch.zeros((inputs.size(0), 1, input_size[1] // stride,
input_size[0] // stride)))
bboxes, labels = self.ssd_target_generator(feat_list,
batch_gt_bboxes,
batch_gt_labels,
input_size)
eye_matrix = torch.eye(self.configer.get('data', 'num_classes'))
labels_target = eye_matrix[labels.view(-1)].view(
inputs.size(0), -1, self.configer.get('data', 'num_classes'))
batch_detections = self.decode(
bboxes, labels_target,
self.ssd_priorbox_layer(feat_list, input_size), self.configer,
input_size)
for j in range(inputs.size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img_bgr = self.blob_helper.tensor2bgr(inputs[j])
self.det_visualizer.vis_default_bboxes(
ori_img_bgr,
self.ssd_priorbox_layer(feat_list, input_size), labels[j])
json_dict = self.__get_info_tree(batch_detections[j],
ori_img_bgr, input_size)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('res', 'vis_conf_thre'))
cv2.imwrite(
os.path.join(vis_dir, '{}_{}_vis.png'.format(i, j)),
image_canvas)
cv2.imshow('main', image_canvas)
cv2.waitKey()
class YOLOv3Test(object):
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.yolo_target_generator = YOLOTargetGenerator(configer)
self.yolo_detection_layer = YOLODetectionLayer(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = RunnerHelper.load_net(self, self.det_net)
self.det_net.eval()
def __test_img(self, image_path, json_path, raw_path, vis_path):
Log.info('Image Path: {}'.format(image_path))
img = ImageHelper.read_image(
image_path,
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
ori_img_bgr = ImageHelper.get_cv2_bgr(img,
mode=self.configer.get(
'data', 'input_mode'))
inputs = self.blob_helper.make_input(img,
input_size=self.configer.get(
'data', 'input_size'),
scale=1.0)
with torch.no_grad():
inputs = inputs.unsqueeze(0).to(self.device)
_, _, detections = self.det_net(inputs)
batch_detections = self.decode(detections, self.configer)
json_dict = self.__get_info_tree(batch_detections[0], ori_img_bgr)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('res', 'vis_conf_thre'))
ImageHelper.save(ori_img_bgr, raw_path)
ImageHelper.save(image_canvas, vis_path)
Log.info('Json Path: {}'.format(json_path))
JsonHelper.save_file(json_dict, json_path)
return json_dict
@staticmethod
def decode(batch_pred_bboxes, configer, input_size):
box_corner = batch_pred_bboxes.new(batch_pred_bboxes.shape)
box_corner[:, :,
0] = batch_pred_bboxes[:, :,
0] - batch_pred_bboxes[:, :, 2] / 2
box_corner[:, :,
1] = batch_pred_bboxes[:, :,
1] - batch_pred_bboxes[:, :, 3] / 2
box_corner[:, :,
2] = batch_pred_bboxes[:, :,
0] + batch_pred_bboxes[:, :, 2] / 2
box_corner[:, :,
3] = batch_pred_bboxes[:, :,
1] + batch_pred_bboxes[:, :, 3] / 2
# clip bounding box
box_corner[:, :, 0::2] = box_corner[:, :, 0::2].clamp(min=0, max=1.0)
box_corner[:, :, 1::2] = box_corner[:, :, 1::2].clamp(min=0, max=1.0)
batch_pred_bboxes[:, :, :4] = box_corner[:, :, :4]
batch_pred_bboxes[:, :, 0::2] *= input_size[0]
batch_pred_bboxes[:, :, 1::2] *= input_size[1]
output = [None for _ in range(len(batch_pred_bboxes))]
for image_i, image_pred in enumerate(batch_pred_bboxes):
# Filter out confidence scores below threshold
conf_mask = (image_pred[:, 4] > configer.get(
'res', 'val_conf_thre')).squeeze()
image_pred = image_pred[conf_mask]
# If none are remaining => process next image
if image_pred.numel() == 0:
continue
# Get score and class with highest confidence
class_conf, class_pred = torch.max(
image_pred[:, 5:5 + configer.get('data', 'num_classes')],
1,
keepdim=True)
# Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)
detections = torch.cat(
(image_pred[:, :5], class_conf.float(), class_pred.float()), 1)
output[image_i] = DetHelper.cls_nms(detections,
labels=class_pred.squeeze(1),
max_threshold=configer.get(
'nms', 'max_threshold'))
return output
def __get_info_tree(self, detections, image_raw, input_size):
height, width, _ = image_raw.shape
json_dict = dict()
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
object_dict = dict()
xmin = x1.cpu().item() / input_size[0] * width
ymin = y1.cpu().item() / input_size[1] * height
xmax = x2.cpu().item() / input_size[0] * width
ymax = y2.cpu().item() / input_size[1] * height
object_dict['bbox'] = [xmin, ymin, xmax, ymax]
object_dict['label'] = int(cls_pred.cpu().item())
object_dict['score'] = float('%.2f' % conf.cpu().item())
object_list.append(object_dict)
json_dict['objects'] = object_list
return json_dict
def debug(self, vis_dir):
count = 0
for i, data_dict in enumerate(self.det_data_loader.get_trainloader()):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
feat_list = list()
for stride in self.configer.get('network', 'stride_list'):
feat_list.append(
torch.zeros((inputs.size(0), 1, input_size[1] // stride,
input_size[0] // stride)))
targets, _, _ = self.yolo_target_generator(feat_list,
batch_gt_bboxes,
batch_gt_labels,
input_size)
targets = targets.to(self.device)
anchors_list = self.configer.get('gt', 'anchors_list')
output_list = list()
be_c = 0
for f_index, anchors in enumerate(anchors_list):
feat_stride = self.configer.get('network',
'stride_list')[f_index]
fm_size = [
int(round(border / feat_stride)) for border in input_size
]
num_c = len(anchors) * fm_size[0] * fm_size[1]
output_list.append(
targets[:, be_c:be_c + num_c].contiguous().view(
targets.size(0), len(anchors), fm_size[1], fm_size[0],
-1).permute(0, 1, 4, 2, 3).contiguous().view(
targets.size(0), -1, fm_size[1], fm_size[0]))
be_c += num_c
batch_detections = self.decode(
self.yolo_detection_layer(output_list)[2], self.configer,
input_size)
for j in range(inputs.size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img_bgr = self.blob_helper.tensor2bgr(inputs[j])
json_dict = self.__get_info_tree(batch_detections[j],
ori_img_bgr, input_size)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('vis', 'obj_threshold'))
cv2.imwrite(
os.path.join(vis_dir, '{}_{}_vis.png'.format(i, j)),
image_canvas)
cv2.imshow('main', image_canvas)
cv2.waitKey()
class SingleShotDetector(object):
"""
The class for Single Shot Detector. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.ssd_target_generator = SSDTargetGenerator(configer)
self.ssd_priorbox_layer = SSDPriorBoxLayer(configer)
self.det_running_score = DetRunningScore(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.data_transformer = DataTransformer(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
self.det_loss = self.det_loss_manager.get_det_loss('ssd_multibox_loss')
def _get_parameters(self):
return self.det_net.parameters()
def warm_lr(self, batch_len):
"""Sets the learning rate
# Adapted from PyTorch Imagenet example:
# https://github.com/pytorch/examples/blob/master/imagenet/main.py
"""
warm_iters = self.configer.get('lr', 'warm')['warm_epoch'] * batch_len
warm_lr = self.configer.get('lr', 'warm')['warm_lr']
if self.configer.get('iters') < warm_iters:
lr_delta = (self.configer.get('lr', 'base_lr') -
warm_lr) * self.configer.get('iters') / warm_iters
lr = warm_lr + lr_delta
for param_group in self.optimizer.param_groups:
param_group['lr'] = lr
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_dict in enumerate(self.train_loader):
if not self.configer.is_empty(
'lr', 'is_warm') and self.configer.get('lr', 'is_warm'):
self.warm_lr(len(self.train_loader))
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
# Change the data type.
inputs = self.module_utilizer.to_device(inputs)
self.data_time.update(time.time() - start_time)
# Forward pass.
feat_list, loc, cls = self.det_net(inputs)
bboxes, labels = self.ssd_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels,
[inputs.size(3), inputs.size(2)])
bboxes, labels = self.module_utilizer.to_device(bboxes, labels)
# Compute the loss of the train batch & backward.
loss = self.det_loss(loc, bboxes, cls, labels)
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
(self.configer.get('iters')) % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for j, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
inputs = self.module_utilizer.to_device(inputs)
input_size = [inputs.size(3), inputs.size(2)]
# Forward pass.
feat_list, loc, cls = self.det_net(inputs)
bboxes, labels = self.ssd_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels, input_size)
bboxes, labels = self.module_utilizer.to_device(bboxes, labels)
# Compute the loss of the val batch.
loss = self.det_loss(loc, bboxes, cls, labels)
self.val_losses.update(loss.item(), inputs.size(0))
batch_detections = SingleShotDetectorTest.decode(
loc, cls, self.ssd_priorbox_layer(feat_list, input_size),
self.configer, input_size)
batch_pred_bboxes = self.__get_object_list(batch_detections)
# batch_pred_bboxes = self._get_gt_object_list(batch_gt_bboxes, batch_gt_labels)
self.det_running_score.update(batch_pred_bboxes,
batch_gt_bboxes, batch_gt_labels)
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.det_net, save_mode='iters')
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
Log.info('Val mAP: {}'.format(self.det_running_score.get_mAP()))
self.det_running_score.reset()
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def _get_gt_object_list(self, batch_gt_bboxes, batch_gt_labels):
batch_pred_bboxes = list()
for i in range(len(batch_gt_bboxes)):
object_list = list()
if batch_gt_bboxes[i].numel() > 0:
for j in range(batch_gt_bboxes[i].size(0)):
object_list.append([
batch_gt_bboxes[i][j][0].item(),
batch_gt_bboxes[i][j][1].item(),
batch_gt_bboxes[i][j][2].item(),
batch_gt_bboxes[i][j][3].item(),
batch_gt_labels[i][j].item(), 1.0
])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
def __get_object_list(self, batch_detections):
batch_pred_bboxes = list()
for idx, detections in enumerate(batch_detections):
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
xmin = x1.cpu().item()
ymin = y1.cpu().item()
xmax = x2.cpu().item()
ymax = y2.cpu().item()
cf = conf.cpu().item()
cls_pred = cls_pred.cpu().item() - 1
object_list.append([
xmin, ymin, xmax, ymax,
int(cls_pred),
float('%.2f' % cf)
])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
def train(self):
cudnn.benchmark = True
if self.configer.get('network',
'resume') is not None and self.configer.get(
'network', 'resume_val'):
self.__val()
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class SingleShotDetector(object):
"""
The class for Single Shot Detector. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
def init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
self.det_loss = self.det_loss_manager.get_det_loss('multibox_loss')
def _get_parameters(self):
return self.det_net.parameters()
def __train(self):
"""
Train function of every epoch during train phase.
"""
if self.configer.get(
'network',
'resume') is not None and self.configer.get('iters') == 0:
self.__val()
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, (inputs, bboxes, labels) in enumerate(self.train_loader):
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs, bboxes, labels = self.module_utilizer.to_device(
inputs, bboxes, labels)
# Forward pass.
loc, cls = self.det_net(inputs)
# Compute the loss of the train batch & backward.
loss = self.det_loss(loc, bboxes, cls, labels)
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
(self.configer.get('iters')) % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for j, (inputs, bboxes, labels) in enumerate(self.val_loader):
# Change the data type.
inputs, bboxes, labels = self.module_utilizer.to_device(
inputs, bboxes, labels)
# Forward pass.
loc, cls = self.det_net(inputs)
# Compute the loss of the val batch.
loss = self.det_loss(loc, bboxes, cls, labels)
self.val_losses.update(loss.item(), inputs.size(0))
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.det_net, metric='iters')
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def train(self):
cudnn.benchmark = True
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class SingleShotDetectorTest(object):
def __init__(self, configer):
self.configer = configer
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.default_boxes = PriorBoxLayer(configer)()
self.device = torch.device('cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
def init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.det_net.eval()
def __test_img(self, image_path, json_path, raw_path, vis_path):
Log.info('Image Path: {}'.format(image_path))
ori_img_rgb = ImageHelper.img2np(ImageHelper.pil_open_rgb(image_path))
ori_img_bgr = ImageHelper.rgb2bgr(ori_img_rgb)
inputs = ImageHelper.resize(ori_img_rgb, tuple(self.configer.get('data', 'input_size')), Image.CUBIC)
inputs = ToTensor()(inputs)
inputs = Normalize(mean=self.configer.get('trans_params', 'mean'),
std=self.configer.get('trans_params', 'std'))(inputs)
with torch.no_grad():
inputs = inputs.unsqueeze(0).to(self.device)
bbox, cls = self.det_net(inputs)
bbox = bbox.cpu().data.squeeze(0)
cls = F.softmax(cls.cpu().squeeze(0), dim=-1).data
boxes, lbls, scores = self.__decode(bbox, cls)
json_dict = self.__get_info_tree(boxes, lbls, scores, ori_img_rgb)
image_canvas = self.det_parser.draw_bboxes(ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('vis', 'conf_threshold'))
cv2.imwrite(vis_path, image_canvas)
cv2.imwrite(raw_path, ori_img_bgr)
Log.info('Json Path: {}'.format(json_path))
JsonHelper.save_file(json_dict, json_path)
return json_dict
def __nms(self, bboxes, scores, mode='union'):
"""Non maximum suppression.
Args:
bboxes(tensor): bounding boxes, sized [N,4].
scores(tensor): bbox scores, sized [N,].
threshold(float): overlap threshold.
mode(str): 'union' or 'min'.
Returns:
keep(tensor): selected indices.
Ref:
https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py
"""
x1 = bboxes[:, 0]
y1 = bboxes[:, 1]
x2 = bboxes[:, 2]
y2 = bboxes[:, 3]
areas = (x2 - x1) * (y2 - y1)
_, order = scores.sort(0, descending=True)
keep = []
while order.numel() > 0:
i = order[0]
keep.append(i)
if order.numel() == 1:
break
xx1 = x1[order[1:]].clamp(min=x1[i])
yy1 = y1[order[1:]].clamp(min=y1[i])
xx2 = x2[order[1:]].clamp(max=x2[i])
yy2 = y2[order[1:]].clamp(max=y2[i])
w = (xx2-xx1).clamp(min=0)
h = (yy2-yy1).clamp(min=0)
inter = w*h
if self.configer.get('nms', 'mode') == 'union':
ovr = inter / (areas[i] + areas[order[1:]] - inter)
elif self.configer.get('nms', 'mode') == 'min':
ovr = inter / areas[order[1:]].clamp(max=areas[i])
else:
raise TypeError('Unknown nms mode: %s.' % mode)
ids = (ovr <= self.configer.get('nms', 'overlap_threshold')).nonzero().squeeze()
if ids.numel() == 0:
break
order = order[ids + 1]
return torch.LongTensor(keep)
def __decode(self, loc, conf):
"""Transform predicted loc/conf back to real bbox locations and class labels.
Args:
loc: (tensor) predicted loc, sized [8732, 4].
conf: (tensor) predicted conf, sized [8732, 21].
Returns:
boxes: (tensor) bbox locations, sized [#obj, 4].
labels: (tensor) class labels, sized [#obj,1].
"""
variances = [0.1, 0.2]
wh = torch.exp(loc[:, 2:] * variances[1]) * self.default_boxes[:, 2:]
cxcy = loc[:, :2] * variances[0] * self.default_boxes[:, 2:] + self.default_boxes[:, :2]
boxes = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 1) # [8732,4]
max_conf, labels = conf.max(1) # [8732,1]
ids = labels.nonzero()
tmp = ids.cpu().numpy()
if tmp.__len__() > 0:
# print('detected %d objs' % tmp.__len__())
ids = ids.squeeze(1) # [#boxes,]
keep = self.__nms(boxes[ids], max_conf[ids])
pred_bboxes = boxes[ids][keep].cpu().numpy()
pred_bboxes = np.clip(pred_bboxes, 0, 1)
pred_labels = labels[ids][keep].cpu().numpy()
pred_confs = max_conf[ids][keep].cpu().numpy()
return pred_bboxes, pred_labels, pred_confs
else:
Log.info('None object detected!')
pred_bboxes = list()
pred_labels = list()
pred_confs = list()
return pred_bboxes, pred_labels, pred_confs
def __get_info_tree(self, box_list, label_list, conf, image_raw):
height, width, _ = image_raw.shape
json_dict = dict()
object_list = list()
for bbox, label, cf in zip(box_list, label_list, conf):
if cf < self.configer.get('vis', 'conf_threshold'):
continue
object_dict = dict()
xmin = bbox[0] * width
xmax = bbox[2] * width
ymin = bbox[1] * height
ymax = bbox[3] * height
object_dict['bbox'] = [xmin, ymin, xmax, ymax]
object_dict['label'] = label - 1
object_dict['score'] = cf
object_list.append(object_dict)
json_dict['objects'] = object_list
return json_dict
def test(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'val/results/det', self.configer.get('dataset'))
test_img = self.configer.get('test_img')
test_dir = self.configer.get('test_dir')
if test_img is None and test_dir is None:
Log.error('test_img & test_dir not exists.')
exit(1)
if test_img is not None and test_dir is not None:
Log.error('Either test_img or test_dir.')
exit(1)
if test_img is not None:
base_dir = os.path.join(base_dir, 'test_img')
filename = test_img.rstrip().split('/')[-1]
json_path = os.path.join(base_dir, 'json', '{}.json'.format('.'.join(filename.split('.')[:-1])))
raw_path = os.path.join(base_dir, 'raw', filename)
vis_path = os.path.join(base_dir, 'vis', '{}_vis.png'.format('.'.join(filename.split('.')[:-1])))
if not os.path.exists(os.path.dirname(json_path)):
os.makedirs(os.path.dirname(json_path))
if not os.path.exists(os.path.dirname(raw_path)):
os.makedirs(os.path.dirname(raw_path))
if not os.path.exists(os.path.dirname(vis_path)):
os.makedirs(os.path.dirname(vis_path))
self.__test_img(test_img, json_path, raw_path, vis_path)
else:
base_dir = os.path.join(base_dir, 'test_dir', test_dir.rstrip('/').split('/')[-1])
if not os.path.exists(base_dir):
os.makedirs(base_dir)
for filename in FileHelper.list_dir(test_dir):
image_path = os.path.join(test_dir, filename)
json_path = os.path.join(base_dir, 'json', '{}.json'.format('.'.join(filename.split('.')[:-1])))
raw_path = os.path.join(base_dir, 'raw', filename)
vis_path = os.path.join(base_dir, 'vis', '{}_vis.png'.format('.'.join(filename.split('.')[:-1])))
if not os.path.exists(os.path.dirname(json_path)):
os.makedirs(os.path.dirname(json_path))
if not os.path.exists(os.path.dirname(raw_path)):
os.makedirs(os.path.dirname(raw_path))
if not os.path.exists(os.path.dirname(vis_path)):
os.makedirs(os.path.dirname(vis_path))
self.__test_img(image_path, json_path, raw_path, vis_path)
def debug(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'vis/results/det', self.configer.get('dataset'), 'debug')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
val_data_loader = self.det_data_loader.get_valloader()
count = 0
for i, (inputs, bboxes, labels) in enumerate(val_data_loader):
for j in range(inputs.size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img_rgb = DeNormalize(mean=self.configer.get('trans_params', 'mean'),
std=self.configer.get('trans_params', 'std'))(inputs[j])
ori_img_rgb = ori_img_rgb.numpy().transpose(1, 2, 0).astype(np.uint8)
ori_img_bgr = cv2.cvtColor(ori_img_rgb, cv2.COLOR_RGB2BGR)
eye_matrix = torch.eye(self.configer.get('data', 'num_classes'))
labels_target = eye_matrix[labels.view(-1)].view(inputs.size(0), -1,
self.configer.get('data', 'num_classes'))
boxes, lbls, scores = self.__decode(bboxes[j], labels_target[j])
json_dict = self.__get_info_tree(boxes, lbls, scores, ori_img_rgb)
image_canvas = self.det_parser.draw_bboxes(ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('vis', 'conf_threshold'))
cv2.imwrite(os.path.join(base_dir, '{}_{}_vis.png'.format(i, j)), image_canvas)
cv2.imshow('main', image_canvas)
cv2.waitKey()
class SingleShotDetectorTest(object):
def __init__(self, configer):
self.configer = configer
self.det_visualizer = DetVisualizer(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.default_boxes = PriorBoxLayer(configer)()
self.device = torch.device('cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
def init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.det_net.eval()
def __test_img(self, image_path, save_path):
Log.info('Image Path: {}'.format(image_path))
image_raw = ImageHelper.cv2_open_bgr(image_path)
inputs = ImageHelper.bgr2rgb(image_raw)
inputs = ImageHelper.resize(inputs, tuple(self.configer.get('data', 'input_size')), Image.CUBIC)
inputs = ToTensor()(inputs)
inputs = Normalize(mean=self.configer.get('trans_params', 'mean'),
std=self.configer.get('trans_params', 'std'))(inputs)
with torch.no_grad():
inputs = inputs.unsqueeze(0).to(self.device)
bbox, cls = self.det_net(inputs)
bbox = bbox.cpu().data.squeeze(0)
cls = F.softmax(cls.cpu().squeeze(0), dim=-1).data
boxes, lbls, scores, has_obj = self.__decode(bbox, cls)
if has_obj:
boxes = boxes.cpu().numpy()
boxes = np.clip(boxes, 0, 1)
lbls = lbls.cpu().numpy()
scores = scores.cpu().numpy()
img_canvas = self.__draw_box(image_raw, boxes, lbls, scores)
else:
# print('None obj detected!')
img_canvas = image_raw
Log.info('Save Path: {}'.format(save_path))
cv2.imwrite(save_path, img_canvas)
# Boxes is within 0-1.
self.__save_json(save_path, boxes, lbls, scores, image_raw)
return image_raw, lbls, scores, boxes, has_obj
def __draw_box(self, img_raw, box_list, label_list, conf):
img_canvas = img_raw.copy()
img_shape = img_canvas.shape
for bbox, label, cf in zip(box_list, label_list, conf):
if cf < self.configer.get('vis', 'conf_threshold'):
continue
xmin = int(bbox[0] * img_shape[1])
xmax = int(bbox[2] * img_shape[1])
ymin = int(bbox[1] * img_shape[0])
ymax = int(bbox[3] * img_shape[0])
class_name = self.configer.get('details', 'name_seq')[label - 1] + str(cf)
c = self.configer.get('details', 'color_list')[label - 1]
cv2.rectangle(img_canvas, (xmin, ymin), (xmax, ymax), color=c, thickness=3)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img_canvas, class_name, (xmin + 5, ymax - 5), font, fontScale=0.5, color=c, thickness=2)
return img_canvas
def __nms(self, bboxes, scores, mode='union'):
"""Non maximum suppression.
Args:
bboxes(tensor): bounding boxes, sized [N,4].
scores(tensor): bbox scores, sized [N,].
threshold(float): overlap threshold.
mode(str): 'union' or 'min'.
Returns:
keep(tensor): selected indices.
Ref:
https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py
"""
x1 = bboxes[:, 0]
y1 = bboxes[:, 1]
x2 = bboxes[:, 2]
y2 = bboxes[:, 3]
areas = (x2 - x1) * (y2 - y1)
_, order = scores.sort(0, descending=True)
keep = []
while order.numel() > 0:
i = order[0]
keep.append(i)
if order.numel() == 1:
break
xx1 = x1[order[1:]].clamp(min=x1[i])
yy1 = y1[order[1:]].clamp(min=y1[i])
xx2 = x2[order[1:]].clamp(max=x2[i])
yy2 = y2[order[1:]].clamp(max=y2[i])
w = (xx2-xx1).clamp(min=0)
h = (yy2-yy1).clamp(min=0)
inter = w*h
if self.configer.get('nms', 'mode') == 'union':
ovr = inter / (areas[i] + areas[order[1:]] - inter)
elif self.configer.get('nms', 'mode') == 'min':
ovr = inter / areas[order[1:]].clamp(max=areas[i])
else:
raise TypeError('Unknown nms mode: %s.' % mode)
ids = (ovr <= self.configer.get('nms', 'overlap_threshold')).nonzero().squeeze()
if ids.numel() == 0:
break
order = order[ids + 1]
return torch.LongTensor(keep)
def __decode(self, loc, conf):
"""Transform predicted loc/conf back to real bbox locations and class labels.
Args:
loc: (tensor) predicted loc, sized [8732, 4].
conf: (tensor) predicted conf, sized [8732, 21].
Returns:
boxes: (tensor) bbox locations, sized [#obj, 4].
labels: (tensor) class labels, sized [#obj,1].
"""
has_obj = False
variances = [0.1, 0.2]
wh = torch.exp(loc[:, 2:] * variances[1]) * self.default_boxes[:, 2:]
cxcy = loc[:, :2] * variances[0] * self.default_boxes[:, 2:] + self.default_boxes[:, :2]
boxes = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 1) # [8732,4]
max_conf, labels = conf.max(1) # [8732,1]
ids = labels.nonzero()
tmp = ids.cpu().numpy()
if tmp.__len__() > 0:
# print('detected %d objs' % tmp.__len__())
ids = ids.squeeze(1) # [#boxes,]
has_obj = True
else:
print('None obj detected!')
return 0, 0, 0, has_obj
keep = self.__nms(boxes[ids], max_conf[ids])
return boxes[ids][keep], labels[ids][keep], max_conf[ids][keep], has_obj
def __save_json(self, save_path, box_list, label_list, conf, image_raw):
file_name = '{}.json'.format(save_path[:-4], ".json")
json_file_stream = open(file_name, 'w')
size = image_raw.shape
json_dict = dict()
object_list = list()
for bbox, label, cf in zip(box_list, label_list, conf):
if cf < self.configer.get('vis', 'conf_threshold'):
continue
object_dict = dict()
xmin = bbox[0] * size[1]
xmax = bbox[2] * size[1]
ymin = bbox[1] * size[0]
ymax = bbox[3] * size[0]
object_dict['bbox'] = [xmin, xmax, ymin, ymax]
object_dict['label'] = label - 1
object_list.append(object_dict)
json_dict['objects'] = object_list
json_file_stream.write(json.dumps(json_dict))
json_file_stream.close()
def test(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'val/results/det', self.configer.get('dataset'))
test_img = self.configer.get('test_img')
test_dir = self.configer.get('test_dir')
if test_img is None and test_dir is None:
Log.error('test_img & test_dir not exists.')
exit(1)
if test_img is not None and test_dir is not None:
Log.error('Either test_img or test_dir.')
exit(1)
if test_img is not None:
base_dir = os.path.join(base_dir, 'test_img')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
filename = test_img.rstrip().split('/')[-1]
save_path = os.path.join(base_dir, filename)
self.__test_img(test_img, save_path)
else:
base_dir = os.path.join(base_dir, 'test_dir', test_dir.rstrip('/').split('/')[-1])
if not os.path.exists(base_dir):
os.makedirs(base_dir)
for filename in self.__list_dir(test_dir):
image_path = os.path.join(test_dir, filename)
save_path = os.path.join(base_dir, filename)
if not os.path.exists(os.path.dirname(save_path)):
os.makedirs(os.path.dirname(save_path))
self.__test_img(image_path, save_path)
def debug(self):
base_dir = os.path.join(self.configer.get('project_dir'),
'vis/results/det', self.configer.get('dataset'), 'debug')
if not os.path.exists(base_dir):
os.makedirs(base_dir)
val_data_loader = self.det_data_loader.get_valloader(SSDDataLoader)
count = 0
for i, (inputs, bboxes, labels) in enumerate(val_data_loader):
for j in range(inputs.size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img = DeNormalize(mean=self.configer.get('trans_params', 'mean'),
std=self.configer.get('trans_params', 'std'))(inputs[j])
ori_img = ori_img.numpy().transpose(1, 2, 0)
image_bgr = cv2.cvtColor(ori_img, cv2.COLOR_RGB2BGR)
eye_matrix = torch.eye(self.configer.get('data', 'num_classes'))
labels_target = eye_matrix[labels.view(-1)].view(inputs.size(0), -1,
self.configer.get('data', 'num_classes'))
boxes, lbls, scores, has_obj = self.__decode(bboxes[j], labels_target[j])
if has_obj:
boxes = boxes.cpu().numpy()
boxes = np.clip(boxes, 0, 1)
lbls = lbls.cpu().numpy()
scores = scores.cpu().numpy()
img_canvas = self.__draw_box(image_bgr, boxes, lbls, scores)
else:
# print('None obj detected!')
img_canvas = image_bgr
# self.det_visualizer.vis_bboxes(paf_avg, image_rgb.astype(np.uint8), name='314{}_{}'.format(i,j))
cv2.imwrite(os.path.join(base_dir, '{}_{}_result.jpg'.format(i, j)), img_canvas)
def __list_dir(self, dir_name):
filename_list = list()
for item in os.listdir(dir_name):
if os.path.isdir(os.path.join(dir_name, item)):
for filename in os.listdir(os.path.join(dir_name, item)):
filename_list.append('{}/{}'.format(item, filename))
else:
filename_list.append(item)
return filename_list
def __init__(self, model_path=None, gpu_id=0):
self.model_path = model_path
self.det_visualizer = DetVisualizer(self.configer)
self.det_model_manager = DetModelManager(self.configer)
self.default_boxes = PriorBoxLayer(self.configer)()
self.det_net = None
class YOLOv3(object):
"""
The class for YOLO v3. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = DetLossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DetDataLoader(configer)
self.yolo_detection_layer = YOLODetectionLayer(configer)
self.yolo_target_generator = YOLOTargetGenerator(configer)
self.det_running_score = DetRunningScore(configer)
self.module_utilizer = ModuleUtilizer(configer)
self.optim_scheduler = OptimScheduler(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = self.module_utilizer.load_net(self.det_net)
self.optimizer, self.scheduler = self.optim_scheduler.init_optimizer(
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
self.det_loss = self.det_loss_manager.get_det_loss('yolov3_loss')
def _get_parameters(self):
lr_1 = []
lr_10 = []
params_dict = dict(self.det_net.named_parameters())
for key, value in params_dict.items():
if 'backbone.' not in key:
lr_10.append(value)
else:
lr_1.append(value)
params = [{
'params': lr_1,
'lr': self.configer.get('lr', 'base_lr')
}, {
'params': lr_10,
'lr': self.configer.get('lr', 'base_lr') * 10.
}]
return params
def warm_lr(self, batch_len):
"""Sets the learning rate
# Adapted from PyTorch Imagenet example:
# https://github.com/pytorch/examples/blob/master/imagenet/main.py
"""
warm_iters = self.configer.get('lr', 'warm')['warm_epoch'] * batch_len
if self.configer.get('iters') < warm_iters:
lr_ratio = (self.configer.get('iters') + 1) / warm_iters
base_lr_list = self.scheduler.get_lr()
for param_group, base_lr in zip(self.optimizer.param_groups,
base_lr_list):
param_group['lr'] = base_lr * lr_ratio
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info('LR: {}'.format([
param_group['lr']
for param_group in self.optimizer.param_groups
]))
def __train(self):
"""
Train function of every epoch during train phase.
"""
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.configer.plus_one('epoch')
self.scheduler.step(self.configer.get('epoch'))
# data_tuple: (inputs, heatmap, maskmap, vecmap)
for i, data_dict in enumerate(self.train_loader):
if not self.configer.is_empty(
'lr', 'is_warm') and self.configer.get('lr', 'is_warm'):
self.warm_lr(len(self.train_loader))
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
self.data_time.update(time.time() - start_time)
# Change the data type.
inputs = self.module_utilizer.to_device(inputs)
# Forward pass.
feat_list, predictions, _ = self.det_net(inputs)
targets, objmask, noobjmask = self.yolo_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels, input_size)
targets, objmask, noobjmask = self.module_utilizer.to_device(
targets, objmask, noobjmask)
# Compute the loss of the train batch & backward.
loss = self.det_loss(predictions, targets, objmask, noobjmask)
self.train_losses.update(loss.item(), inputs.size(0))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.configer.plus_one('iters')
# Print the log info & reset the states.
if self.configer.get('iters') % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.configer.get('epoch'),
self.configer.get('iters'),
self.configer.get('solver', 'display_iter'),
self.scheduler.get_lr(),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
# Check to val the current model.
if self.val_loader is not None and \
(self.configer.get('iters')) % self.configer.get('solver', 'test_interval') == 0:
self.__val()
def __val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for i, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
input_size = [inputs.size(3), inputs.size(2)]
# Forward pass.
inputs = self.module_utilizer.to_device(inputs)
feat_list, predictions, detections = self.det_net(inputs)
targets, objmask, noobjmask = self.yolo_target_generator(
feat_list, batch_gt_bboxes, batch_gt_labels, input_size)
targets, objmask, noobjmask = self.module_utilizer.to_device(
targets, objmask, noobjmask)
# Compute the loss of the val batch.
loss = self.det_loss(predictions, targets, objmask, noobjmask)
self.val_losses.update(loss.item(), inputs.size(0))
batch_detections = YOLOv3Test.decode(detections, self.configer)
batch_pred_bboxes = self.__get_object_list(
batch_detections, input_size)
self.det_running_score.update(batch_pred_bboxes,
batch_gt_bboxes, batch_gt_labels)
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.module_utilizer.save_net(self.det_net, save_mode='iters')
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
Log.info('Val mAP: {}'.format(self.det_running_score.get_mAP()))
self.det_running_score.reset()
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def __get_object_list(self, batch_detections, input_size):
batch_pred_bboxes = list()
for idx, detections in enumerate(batch_detections):
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_conf, cls_pred in detections:
xmin = x1.cpu().item() * input_size[0]
ymin = y1.cpu().item() * input_size[1]
xmax = x2.cpu().item() * input_size[0]
ymax = y2.cpu().item() * input_size[1]
cf = conf.cpu().item()
cls_pred = cls_pred.cpu().item()
object_list.append([
xmin, ymin, xmax, ymax,
int(cls_pred),
float('%.2f' % cf)
])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
def train(self):
cudnn.benchmark = True
if self.configer.get('network',
'resume') is not None and self.configer.get(
'network', 'resume_val'):
self.__val()
while self.configer.get('epoch') < self.configer.get(
'solver', 'max_epoch'):
self.__train()
if self.configer.get('epoch') == self.configer.get(
'solver', 'max_epoch'):
break
class FastRCNNTest(object):
def __init__(self, configer):
self.configer = configer
self.blob_helper = BlobHelper(configer)
self.det_visualizer = DetVisualizer(configer)
self.det_parser = DetParser(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.roi_sampler = FRROISampler(configer)
self.rpn_target_generator = RPNTargetAssigner(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.fr_roi_generator = FRROIGenerator(configer)
self.device = torch.device(
'cpu' if self.configer.get('gpu') is None else 'cuda')
self.det_net = None
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = RunnerHelper.load_net(self, self.det_net)
self.det_net.eval()
def __test_img(self, image_path, json_path, raw_path, vis_path):
Log.info('Image Path: {}'.format(image_path))
image = ImageHelper.read_image(
image_path,
tool=self.configer.get('data', 'image_tool'),
mode=self.configer.get('data', 'input_mode'))
ori_img_bgr = ImageHelper.get_cv2_bgr(image,
mode=self.configer.get(
'data', 'input_mode'))
width, height = ImageHelper.get_size(image)
scale1 = self.configer.get('test', 'resize_bound')[0] / min(
width, height)
scale2 = self.configer.get('test', 'resize_bound')[1] / max(
width, height)
scale = min(scale1, scale2)
inputs = self.blob_helper.make_input(image, scale=scale)
b, c, h, w = inputs.size()
border_wh = [w, h]
if self.configer.exists('test', 'fit_stride'):
stride = self.configer.get('test', 'fit_stride')
pad_w = 0 if (w % stride == 0) else stride - (w % stride) # right
pad_h = 0 if (h % stride == 0) else stride - (h % stride) # down
expand_image = torch.zeros(
(b, c, h + pad_h, w + pad_w)).to(inputs.device)
expand_image[:, :, 0:h, 0:w] = inputs
inputs = expand_image
data_dict = dict(
img=inputs,
meta=DataContainer([[
dict(ori_img_size=ImageHelper.get_size(ori_img_bgr),
aug_img_size=border_wh,
img_scale=scale,
input_size=[inputs.size(3),
inputs.size(2)])
]],
cpu_only=True))
with torch.no_grad():
# Forward pass.
test_group = self.det_net(data_dict)
test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group
batch_detections = self.decode(test_roi_locs, test_roi_scores,
test_indices_and_rois, test_rois_num,
self.configer,
DCHelper.tolist(data_dict['meta']))
json_dict = self.__get_info_tree(batch_detections[0],
ori_img_bgr,
scale=scale)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('res', 'vis_conf_thre'))
cv2.imwrite(vis_path, image_canvas)
cv2.imwrite(raw_path, ori_img_bgr)
Log.info('Json Path: {}'.format(json_path))
JsonHelper.save_file(json_dict, json_path)
return json_dict
@staticmethod
def decode(roi_locs, roi_scores, indices_and_rois, test_rois_num, configer,
metas):
indices_and_rois = indices_and_rois
num_classes = configer.get('data', 'num_classes')
mean = torch.Tensor(configer.get(
'roi', 'loc_normalize_mean')).repeat(num_classes)[None]
std = torch.Tensor(configer.get(
'roi', 'loc_normalize_std')).repeat(num_classes)[None]
mean = mean.to(roi_locs.device)
std = std.to(roi_locs.device)
roi_locs = (roi_locs * std + mean)
roi_locs = roi_locs.contiguous().view(-1, num_classes, 4)
# roi_locs = roi_locs[:,:, [1, 0, 3, 2]]
rois = indices_and_rois[:, 1:]
rois = rois.contiguous().view(-1, 1, 4).expand_as(roi_locs)
wh = torch.exp(roi_locs[:, :, 2:]) * (rois[:, :, 2:] - rois[:, :, :2])
cxcy = roi_locs[:, :, :2] * (rois[:, :, 2:] - rois[:, :, :2]) + (
rois[:, :, :2] + rois[:, :, 2:]) / 2
dst_bbox = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 2) # [b, 8732,4]
if configer.get('phase') != 'debug':
cls_prob = F.softmax(roi_scores, dim=1)
else:
cls_prob = roi_scores
cls_label = torch.LongTensor([i for i in range(num_classes)])\
.contiguous().view(1, num_classes).repeat(indices_and_rois.size(0), 1).to(roi_locs.device)
output = [None for _ in range(test_rois_num.size(0))]
start_index = 0
for i in range(test_rois_num.size(0)):
# batch_index = (indices_and_rois[:, 0] == i).nonzero().contiguous().view(-1,)
# tmp_dst_bbox = dst_bbox[batch_index]
# tmp_cls_prob = cls_prob[batch_index]
# tmp_cls_label = cls_label[batch_index]
tmp_dst_bbox = dst_bbox[start_index:start_index + test_rois_num[i]]
# clip bounding box
tmp_dst_bbox[:, :, 0::2] = tmp_dst_bbox[:, :, 0::2].clamp(
min=0, max=metas[i]['border_size'][0] - 1)
tmp_dst_bbox[:, :, 1::2] = tmp_dst_bbox[:, :, 1::2].clamp(
min=0, max=metas[i]['border_size'][1] - 1)
tmp_cls_prob = cls_prob[start_index:start_index + test_rois_num[i]]
tmp_cls_label = cls_label[start_index:start_index +
test_rois_num[i]]
start_index += test_rois_num[i]
mask = (tmp_cls_prob > configer.get(
'res', 'val_conf_thre')) & (tmp_cls_label > 0)
tmp_dst_bbox = tmp_dst_bbox[mask].contiguous().view(-1, 4)
if tmp_dst_bbox.numel() == 0:
continue
tmp_cls_prob = tmp_cls_prob[mask].contiguous().view(
-1, ).unsqueeze(1)
tmp_cls_label = tmp_cls_label[mask].contiguous().view(
-1, ).unsqueeze(1)
valid_preds = torch.cat(
(tmp_dst_bbox, tmp_cls_prob.float(), tmp_cls_label.float()), 1)
output[i] = DetHelper.cls_nms(valid_preds,
labels=valid_preds[:, 5],
max_threshold=configer.get(
'nms', 'max_threshold'))
return output
def __get_info_tree(self, detections, image_raw, scale=1.0):
height, width, _ = image_raw.shape
json_dict = dict()
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
object_dict = dict()
xmin = min(x1.cpu().item() / scale, width - 1)
ymin = min(y1.cpu().item() / scale, height - 1)
xmax = min(x2.cpu().item() / scale, width - 1)
ymax = min(y2.cpu().item() / scale, height - 1)
object_dict['bbox'] = [xmin, ymin, xmax, ymax]
object_dict['label'] = int(cls_pred.cpu().item()) - 1
object_dict['score'] = float('%.2f' % conf.cpu().item())
object_list.append(object_dict)
json_dict['objects'] = object_list
return json_dict
def debug(self, vis_dir):
count = 0
for i, data_dict in enumerate(self.det_data_loader.get_trainloader()):
feat_list = list()
input_size = data_dict['meta'][0]['input_size']
for stride in self.configer.get('rpn', 'stride_list'):
feat_list.append(
torch.zeros(
(data_dict['img'].size(0), 1, input_size[1] // stride,
input_size[0] // stride)))
gt_rpn_locs, gt_rpn_labels = self.rpn_target_generator(
feat_list, data_dict['bboxes'], data_dict['meta'])
eye_matrix = torch.eye(2)
gt_rpn_labels[gt_rpn_labels == -1] = 0
gt_rpn_scores = eye_matrix[gt_rpn_labels.view(-1)].view(
data_dict['img'].size(0), -1, 2)
test_indices_and_rois, _ = self.fr_roi_generator(
feat_list, gt_rpn_locs, gt_rpn_scores,
self.configer.get('rpn', 'n_test_pre_nms'),
self.configer.get('rpn', 'n_test_post_nms'), data_dict['meta'])
sample_rois, gt_roi_locs, gt_roi_labels = self.roi_sampler(
test_indices_and_rois, data_dict['bboxes'],
data_dict['labels'], data_dict['meta'])
self.det_visualizer.vis_rois(data_dict['img'],
sample_rois[gt_roi_labels > 0])
gt_cls_roi_locs = torch.zeros(
(gt_roi_locs.size(0), self.configer.get('data',
'num_classes'), 4))
gt_cls_roi_locs[torch.arange(0, sample_rois.size(0)).long(),
gt_roi_labels.long()] = gt_roi_locs
gt_cls_roi_locs = gt_cls_roi_locs.contiguous().view(
-1, 4 * self.configer.get('data', 'num_classes'))
eye_matrix = torch.eye(self.configer.get('data', 'num_classes'))
gt_roi_scores = eye_matrix[gt_roi_labels.view(-1)].view(
gt_roi_labels.size(0),
self.configer.get('data', 'num_classes'))
test_rois_num = torch.zeros((len(data_dict['bboxes']), )).long()
for batch_id in range(len(data_dict['bboxes'])):
batch_index = (
sample_rois[:, 0] == batch_id).nonzero().contiguous().view(
-1, )
test_rois_num[batch_id] = batch_index.numel()
batch_detections = FastRCNNTest.decode(gt_cls_roi_locs,
gt_roi_scores, sample_rois,
test_rois_num,
self.configer,
data_dict['meta'])
for j in range(data_dict['img'].size(0)):
count = count + 1
if count > 20:
exit(1)
ori_img_bgr = self.blob_helper.tensor2bgr(data_dict['img'][j])
self.det_visualizer.vis_default_bboxes(
ori_img_bgr, self.fr_priorbox_layer(feat_list, input_size),
gt_rpn_labels[j])
json_dict = self.__get_info_tree(batch_detections[j],
ori_img_bgr)
image_canvas = self.det_parser.draw_bboxes(
ori_img_bgr.copy(),
json_dict,
conf_threshold=self.configer.get('res', 'vis_conf_thre'))
cv2.imwrite(
os.path.join(vis_dir, '{}_{}_vis.png'.format(i, j)),
image_canvas)
cv2.imshow('main', image_canvas)
cv2.waitKey()
class SingleShotDetectorDeploy(object):
def __init__(self, model_path=None, gpu_id=0):
self.model_path = model_path
self.det_visualizer = DetVisualizer(self.configer)
self.det_model_manager = DetModelManager(self.configer)
self.default_boxes = PriorBoxLayer(self.configer)()
self.det_net = None
def _init_model(self, model_path, gpu_id):
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
self.det_net = self.det_model_manager.object_detector()
self.det_net = nn.DataParallel(self.det_net).cuda()
if model_path is not None:
model_dict = torch.load(model_path)
self.det_net.load_state_dict(model_dict['state_dict'])
self.configer = Configer(config_dict=model_dict['config_dict'])
else:
Log.error('Model Path is not existed.')
exit(1)
self.det_net.eval()
def inference(self, image_rgb):
image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)
inputs = cv2.resize(image_rgb,
tuple(self.configer.get('data', 'input_size')))
inputs = ToTensor()(inputs)
inputs = Normalize(mean=self.configer.get('trans_params', 'mean'),
std=self.configer.get('trans_params',
'std'))(inputs)
inputs = Variable(inputs.unsqueeze(0).cuda(), volatile=True)
bbox, cls = self.det_net(inputs)
bbox = bbox.cpu().data.squeeze(0)
cls = F.softmax(cls.cpu().squeeze(0), dim=-1).data
boxes, lbls, scores, has_obj = self.__decode(bbox, cls)
if has_obj:
boxes = boxes.cpu().numpy()
boxes = np.clip(boxes, 0, 1)
lbls = lbls.cpu().numpy()
scores = scores.cpu().numpy()
img_shape = image_bgr.shape
for i in range(len(boxes)):
boxes[i][0] = int(boxes[i][0] * img_shape[1])
boxes[i][2] = int(boxes[i][2] * img_shape[1])
boxes[i][1] = int(boxes[i][1] * img_shape[0])
boxes[i][3] = int(boxes[i][3] * img_shape[0])
img_canvas = self.__draw_box(image_bgr, boxes, lbls, scores)
# if is_save_txt:
# self.__save_txt(save_path, boxes, lbls, scores, img_size)
else:
# print('None obj detected!')
img_canvas = image_bgr
# Boxes is within 0-1.
return img_canvas, lbls, scores, boxes, has_obj
def __draw_box(self, img_raw, box_list, label_list, conf):
img_canvas = img_raw.copy()
for bbox, label, cf in zip(box_list, label_list, conf):
if cf < self.configer.get('vis', 'conf_threshold'):
continue
class_name = self.configer.get('details',
'name_seq')[label - 1] + str(cf)
c = self.configer.get('details', 'color_list')[label - 1]
cv2.rectangle(
img_canvas,
(max(0, int(bbox[0] - 10)), max(0, int(bbox[1] - 10))),
(min(img_canvas.shape[1], int(bbox[2] + 10)),
min(img_canvas.shape[0], int(bbox[3] + 10))),
color=c,
thickness=3)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img_canvas,
class_name, (int(bbox[0] + 5), int(bbox[3] - 5)),
font,
fontScale=0.5,
color=c,
thickness=2)
return img_canvas
def __nms(self, bboxes, scores, mode='union'):
"""Non maximum suppression.
Args:
bboxes(tensor): bounding boxes, sized [N,4].
scores(tensor): bbox scores, sized [N,].
threshold(float): overlap threshold.
mode(str): 'union' or 'min'.
Returns:
keep(tensor): selected indices.
Ref:
https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/nms/py_cpu_nms.py
"""
x1 = bboxes[:, 0]
y1 = bboxes[:, 1]
x2 = bboxes[:, 2]
y2 = bboxes[:, 3]
areas = (x2 - x1) * (y2 - y1)
_, order = scores.sort(0, descending=True)
keep = []
while order.numel() > 0:
i = order[0]
keep.append(i)
if order.numel() == 1:
break
xx1 = x1[order[1:]].clamp(min=x1[i])
yy1 = y1[order[1:]].clamp(min=y1[i])
xx2 = x2[order[1:]].clamp(max=x2[i])
yy2 = y2[order[1:]].clamp(max=y2[i])
w = (xx2 - xx1).clamp(min=0)
h = (yy2 - yy1).clamp(min=0)
inter = w * h
if self.configer.get('nms', 'mode') == 'union':
ovr = inter / (areas[i] + areas[order[1:]] - inter)
elif self.configer.get('nms', 'mode') == 'min':
ovr = inter / areas[order[1:]].clamp(max=areas[i])
else:
raise TypeError('Unknown nms mode: %s.' % mode)
ids = (ovr <= self.configer.get(
'nms', 'overlap_threshold')).nonzero().squeeze()
if ids.numel() == 0:
break
order = order[ids + 1]
return torch.LongTensor(keep)
def __decode(self, loc, conf):
"""Transform predicted loc/conf back to real bbox locations and class labels.
Args:
loc: (tensor) predicted loc, sized [8732, 4].
conf: (tensor) predicted conf, sized [8732, 21].
Returns:
boxes: (tensor) bbox locations, sized [#obj, 4].
labels: (tensor) class labels, sized [#obj,1].
"""
has_obj = False
variances = [0.1, 0.2]
wh = torch.exp(loc[:, 2:] * variances[1]) * self.default_boxes[:, 2:]
cxcy = loc[:, :2] * variances[
0] * self.default_boxes[:, 2:] + self.default_boxes[:, :2]
boxes = torch.cat([cxcy - wh / 2, cxcy + wh / 2], 1) # [8732,4]
max_conf, labels = conf.max(1) # [8732,1]
ids = labels.nonzero()
tmp = ids.cpu().numpy()
if tmp.__len__() > 0:
# print('detected %d objs' % tmp.__len__())
ids = ids.squeeze(1) # [#boxes,]
has_obj = True
else:
print('None obj detected!')
return 0, 0, 0, has_obj
keep = self.__nms(boxes[ids], max_conf[ids])
return boxes[ids][keep], labels[ids][keep], max_conf[ids][
keep], has_obj
class FasterRCNN(object):
"""
The class for Single Shot Detector. Include train, val, test & predict.
"""
def __init__(self, configer):
self.configer = configer
self.batch_time = AverageMeter()
self.data_time = AverageMeter()
self.train_losses = AverageMeter()
self.val_losses = AverageMeter()
self.det_visualizer = DetVisualizer(configer)
self.det_loss_manager = LossManager(configer)
self.det_model_manager = DetModelManager(configer)
self.det_data_loader = DataLoader(configer)
self.fr_priorbox_layer = FRPriorBoxLayer(configer)
self.det_running_score = DetRunningScore(configer)
self.det_net = None
self.train_loader = None
self.val_loader = None
self.optimizer = None
self.scheduler = None
self.runner_state = dict()
self._init_model()
def _init_model(self):
self.det_net = self.det_model_manager.object_detector()
self.det_net = RunnerHelper.load_net(self, self.det_net)
self.optimizer, self.scheduler = Trainer.init(self,
self._get_parameters())
self.train_loader = self.det_data_loader.get_trainloader()
self.val_loader = self.det_data_loader.get_valloader()
def _get_parameters(self):
lr_1 = []
lr_2 = []
params_dict = dict(self.det_net.named_parameters())
for key, value in params_dict.items():
if value.requires_grad:
if 'bias' in key:
lr_2.append(value)
else:
lr_1.append(value)
params = [{
'params': lr_1,
'lr': self.configer.get('lr', 'base_lr')
}, {
'params': lr_2,
'lr': self.configer.get('lr', 'base_lr') * 2.,
'weight_decay': 0
}]
return params
def train(self):
"""
Train function of every epoch during train phase.
"""
self.det_net.train()
start_time = time.time()
# Adjust the learning rate after every epoch.
self.runner_state['epoch'] += 1
for i, data_dict in enumerate(self.train_loader):
Trainer.update(self)
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
metas = data_dict['meta']
data_dict['bboxes'] = DCHelper.todc(
batch_gt_bboxes,
gpu_list=self.configer.get('gpu'),
cpu_only=True)
data_dict['labels'] = DCHelper.todc(
batch_gt_labels,
gpu_list=self.configer.get('gpu'),
cpu_only=True)
data_dict['meta'] = DCHelper.todc(
metas, gpu_list=self.configer.get('gpu'), cpu_only=True)
self.data_time.update(time.time() - start_time)
# Forward pass.
loss = self.det_net(data_dict)
loss = loss.mean()
self.train_losses.update(loss.item(), data_dict['img'].size(0))
self.optimizer.zero_grad()
loss.backward()
RunnerHelper.clip_grad(self.det_net, 10.)
self.optimizer.step()
# Update the vars of the train phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
self.runner_state['iters'] += 1
# Print the log info & reset the states.
if self.runner_state['iters'] % self.configer.get(
'solver', 'display_iter') == 0:
Log.info(
'Train Epoch: {0}\tTrain Iteration: {1}\t'
'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t'
'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n'
'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'
.format(self.runner_state['epoch'],
self.runner_state['iters'],
self.configer.get('solver', 'display_iter'),
RunnerHelper.get_lr(self.optimizer),
batch_time=self.batch_time,
data_time=self.data_time,
loss=self.train_losses))
self.batch_time.reset()
self.data_time.reset()
self.train_losses.reset()
if self.configer.get('lr', 'metric') == 'iters' \
and self.runner_state['iters'] == self.configer.get('solver', 'max_iters'):
break
# Check to val the current model.
if self.runner_state['iters'] % self.configer.get(
'solver', 'test_interval') == 0:
self.val()
def val(self):
"""
Validation function during the train phase.
"""
self.det_net.eval()
start_time = time.time()
with torch.no_grad():
for j, data_dict in enumerate(self.val_loader):
inputs = data_dict['img']
batch_gt_bboxes = data_dict['bboxes']
batch_gt_labels = data_dict['labels']
metas = data_dict['meta']
data_dict['bboxes'] = DCHelper.todc(
batch_gt_bboxes,
gpu_list=self.configer.get('gpu'),
cpu_only=True)
data_dict['labels'] = DCHelper.todc(
batch_gt_labels,
gpu_list=self.configer.get('gpu'),
cpu_only=True)
data_dict['meta'] = DCHelper.todc(
metas, gpu_list=self.configer.get('gpu'), cpu_only=True)
# Forward pass.
inputs = RunnerHelper.to_device(self, inputs)
loss, test_group = self.det_net(data_dict)
# Compute the loss of the train batch & backward.
loss = loss.mean()
self.val_losses.update(loss.item(), inputs.size(0))
test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group
batch_detections = FastRCNNTest.decode(test_roi_locs,
test_roi_scores,
test_indices_and_rois,
test_rois_num,
self.configer, metas)
batch_pred_bboxes = self.__get_object_list(batch_detections)
self.det_running_score.update(batch_pred_bboxes,
batch_gt_bboxes, batch_gt_labels)
# Update the vars of the val phase.
self.batch_time.update(time.time() - start_time)
start_time = time.time()
RunnerHelper.save_net(self,
self.det_net,
iters=self.runner_state['iters'])
# Print the log info & reset the states.
Log.info(
'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t'
'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time,
loss=self.val_losses))
Log.info('Val mAP: {}\n'.format(self.det_running_score.get_mAP()))
self.det_running_score.reset()
self.batch_time.reset()
self.val_losses.reset()
self.det_net.train()
def __get_object_list(self, batch_detections):
batch_pred_bboxes = list()
for idx, detections in enumerate(batch_detections):
object_list = list()
if detections is not None:
for x1, y1, x2, y2, conf, cls_pred in detections:
xmin = x1.cpu().item()
ymin = y1.cpu().item()
xmax = x2.cpu().item()
ymax = y2.cpu().item()
cf = conf.cpu().item()
cls_pred = int(cls_pred.cpu().item()) - 1
object_list.append(
[xmin, ymin, xmax, ymax, cls_pred,
float('%.2f' % cf)])
batch_pred_bboxes.append(object_list)
return batch_pred_bboxes
