def draw_bbox_on_lidar_top(top, boxes3d, one_frame_tag):
path = os.path.join(config.cfg.LOG_DIR, 'test', 'top',
'%s.png' % one_frame_tag.replace('/', '_'))
top_image = data.draw_top_image(top)
top_image = data.draw_box3d_on_top(top_image, boxes3d, color=(0, 0, 80))
cv2.imwrite(path, top_image)
print('write %s finished' % path)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='',loss:tuple =None,
frame_tag='unknown_tag',is_train_mode=False):
font = cv2.FONT_HERSHEY_SIMPLEX
text_pos = (5, 25)
self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas)
# origin top view
self.top_image = data.draw_top_image(self.top_view[0])
top_view_log = self.top_image.copy()
# add text on origin
text = frame_tag
cv2.putText(top_view_log, text, text_pos, font, 0.3, (0, 255, 100), 0, cv2.LINE_AA)
if log_rpn:
rpn_img = self.log_rpn(step=step, scope_name=scope_name, is_train_mode=is_train_mode, tensor_board=False)
top_view_log = np.concatenate((top_view_log, rpn_img), 1)
# prediction on top
predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d)
# add fusion loss text
text = ''
if loss != None: text += 'loss c: %6f r: %6f' % loss
cv2.putText(predict_top_view, text, text_pos, font, 0.4, (0, 255, 100), 0, cv2.LINE_AA)
# concatenate top_view_log and final prediction
top_view_log = np.concatenate((top_view_log, predict_top_view), 1)
# new_size = (top_view_log.shape[1] // 2, top_view_log.shape[0] // 2)
# top_view_log = cv2.resize(top_view_log, new_size)
self.summary_image(top_view_log, scope_name + '/top_view', step=step)
# prediction on rgb
text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)]
prediction_on_rgb = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables)
self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name=''):
self.top_image = data.draw_top_image(self.top_view[0])
self.top_image = self.top_image_padding(self.top_image)
if log_rpn: self.log_rpn(step=step, scope_name=scope_name)
self.log_fusion_net_detail(log_subdir, self.fuse_probs,
self.fuse_deltas)
text_lables = [
'No.%d class:1 prob: %.4f' % (i, prob)
for i, prob in enumerate(self.probs)
]
predict_camera_view = nud.draw_box3d_on_camera(self.rgb_image[0],
self.boxes3d,
text_lables=text_lables)
new_size = (predict_camera_view.shape[1] // 2,
predict_camera_view.shape[0] // 2)
predict_camera_view = cv2.resize(predict_camera_view, new_size)
# nud.imsave('predict_camera_view' , predict_camera_view, log_subdir)
self.summary_image(predict_camera_view,
scope_name + '/predict_camera_view',
step=step)
predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d)
# nud.imsave('predict_top_view' , predict_top_view, log_subdir)
self.summary_image(predict_top_view,
scope_name + '/predict_top_view',
step=step)
def predict_log(self,
log_subdir,
log_rpn=False,
step=None,
scope_name='',
loss: tuple = None,
frame_tag='unknown_tag',
is_train_mode=False):
font = cv2.FONT_HERSHEY_SIMPLEX
text_pos = (5, 25)
self.log_fusion_net_detail(log_subdir, self.fuse_probs,
self.fuse_deltas)
# origin top view
self.top_image = data.draw_top_image(self.batch_top_view[0])
top_view_log = self.top_image.copy()
# add text on origin
text = frame_tag
cv2.putText(top_view_log, text, text_pos, font, 0.5, (0, 255, 100), 0,
cv2.LINE_AA)
if log_rpn:
rpn_img = self.log_rpn(step=step,
scope_name=scope_name,
tensor_board=False,
draw_rpn_target=False)
top_view_log = np.concatenate((top_view_log, rpn_img), 1)
# all prediction on top
probs, boxes3d = rcnn_nms(self.fuse_probs,
self.fuse_deltas,
self.rois3d,
score_threshold=0.)
fusion_proposal_top = data.draw_box3d_on_top(self.top_image, boxes3d)
prediction_top = data.draw_box3d_on_top(self.top_image, self.boxes3d)
# add fusion loss text
text = ''
if loss != None: text += 'loss c: %6f r: %6f' % loss
cv2.putText(fusion_proposal_top, text, text_pos, font, 0.5,
(0, 255, 100), 0, cv2.LINE_AA)
# concatenate top_view_log and final prediction
top_view_log = np.concatenate((top_view_log, fusion_proposal_top), 1)
top_view_log = np.concatenate((top_view_log, prediction_top), 1)
self.summary_image(top_view_log, scope_name + '/top_view', step=step)
# prediction on rgb
text_lables = [
'No.%d class:1 prob: %.4f' % (i, prob)
for i, prob in enumerate(self.probs)
]
prediction_on_rgb = nud.draw_box3d_on_camera(self.batch_rgb_images[0],
self.boxes3d,
text_lables=text_lables)
self.summary_image(prediction_on_rgb,
scope_name + '/prediction_on_rgb',
step=step)
def visualize_result(tag, top_view, rgb, boxes3d, probs, gt_boxes3d=[]):
top_image = data.draw_top_image(top_view)
top_image = top_image_padding(top_image)
text_lables = [
'No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(probs)
]
predict_camera_view = nud.draw_box3d_on_camera(rgb,
boxes3d,
text_lables=text_lables)
predict_top_view = data.draw_box3d_on_top(top_image, boxes3d)
# draw gt on camera and top view:
if len(gt_boxes3d) > 0: # array size > 1 cannot directly used in if
predict_top_view = data.draw_box3d_on_top(predict_top_view,
gt_boxes3d,
color=(0, 0, 255))
predict_camera_view = draw_box3d_on_camera(predict_camera_view,
gt_boxes3d,
color=(0, 0, 255))
new_size = (predict_camera_view.shape[1] // 2,
predict_camera_view.shape[0] // 2)
predict_camera_view = cv2.resize(predict_camera_view, new_size)
cv2.imwrite(os.path.join(args.target_dir, 'image', tag + 'rgb_.png'),
predict_camera_view)
cv2.imwrite(os.path.join(args.target_dir, 'image', tag + 'top_.png'),
predict_top_view)
def predict_log(self,
log_subdir,
log_rpn=False,
step=None,
scope_name='',
loss: tuple = None,
frame_tag='unknown_tag',
is_train_mode=False):
font = cv2.FONT_HERSHEY_SIMPLEX
text_pos = (5, 25)
self.log_fusion_net_detail(log_subdir, self.fuse_probs,
self.fuse_deltas)
# origin top view
self.top_image = data.draw_top_image(self.top_view[0])
top_view_log = self.top_image.copy()
# add text on origin
text = frame_tag
cv2.putText(top_view_log, text, text_pos, font, 0.3, (0, 255, 100), 0,
cv2.LINE_AA)
if log_rpn:
rpn_img = self.log_rpn(step=step,
scope_name=scope_name,
is_train_mode=is_train_mode,
tensor_board=False)
top_view_log = np.concatenate((top_view_log, rpn_img), 1)
# prediction on top
predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d)
# add fusion loss text
text = ''
if loss != None: text += 'loss c: %6f r: %6f' % loss
cv2.putText(predict_top_view, text, text_pos, font, 0.4, (0, 255, 100),
0, cv2.LINE_AA)
# concatenate top_view_log and final prediction
top_view_log = np.concatenate((top_view_log, predict_top_view), 1)
# new_size = (top_view_log.shape[1] // 2, top_view_log.shape[0] // 2)
# top_view_log = cv2.resize(top_view_log, new_size)
self.summary_image(top_view_log, scope_name + '/top_view', step=step)
# prediction on rgb
text_lables = [
'No.%d class:1 prob: %.4f' % (i, prob)
for i, prob in enumerate(self.probs)
]
prediction_on_rgb = nud.draw_box3d_on_camera(self.rgb_image[0],
self.boxes3d,
text_lables=text_lables)
self.summary_image(prediction_on_rgb,
scope_name + '/prediction_on_rgb',
step=step)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name=''):
self.top_image = data.draw_top_image(self.top_view[0])
self.top_image = self.top_image_padding(self.top_image)
if log_rpn: self.log_rpn(step=step ,scope_name=scope_name)
self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas)
text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)]
predict_camera_view = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables)
new_size = (predict_camera_view.shape[1] // 2, predict_camera_view.shape[0] // 2)
predict_camera_view = cv2.resize(predict_camera_view, new_size)
# nud.imsave('predict_camera_view' , predict_camera_view, log_subdir)
self.summary_image(predict_camera_view, scope_name + '/predict_camera_view', step=step)
predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d)
# nud.imsave('predict_top_view' , predict_top_view, log_subdir)
self.summary_image(predict_top_view, scope_name + '/predict_top_view', step=step)
def log_image(self, step, frame_tag, prefix, summary_writer):
font = cv2.FONT_HERSHEY_SIMPLEX
self.top_image = data.draw_top_image(self.batch_top_view[0])
top_view_log = self.top_image.copy()
# add text on origin
text = frame_tag
for i, line in enumerate(text.split("/")):
y = 25 + i * 20
cv2.putText(top_view_log, line, (5, y), font, 0.5, (0, 255, 100),
0, cv2.LINE_AA)
# draw groundtruth on topview
for gt_bbox in self.batch_gt_boxes3d[:]:
topview = net.processing.boxes3d.draw_box3d_on_top(
top_view_log, gt_bbox)
self.summary_image(topview,
prefix + '/top_view',
step=step,
summary_writer=summary_writer)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='',loss:tuple =None,
frame_tag='unknown_tag',is_train_mode=False):
font = cv2.FONT_HERSHEY_SIMPLEX
text_pos = (5, 25)
self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas)
# origin top view
self.top_image = data.draw_top_image(self.batch_top_view[0])
top_view_log = self.top_image.copy()
# add text on origin
text = frame_tag
cv2.putText(top_view_log, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA)
if log_rpn:
rpn_img = self.log_rpn(step=step, scope_name=scope_name, tensor_board=False,draw_rpn_target=False)
top_view_log = np.concatenate((top_view_log, rpn_img), 1)
# all prediction on top
probs, boxes3d = rcnn_nms(self.fuse_probs, self.fuse_deltas,self.rois3d, score_threshold=0.)
fusion_proposal_top = data.draw_box3d_on_top(self.top_image, boxes3d,scores=probs,thickness=0)
prediction_top = data.draw_box3d_on_top(self.top_image, self.boxes3d, scores=self.probs,
thickness=0)
# add fusion loss text
text = ''
if loss != None: text += 'loss c: %6f r: %6f' % loss
cv2.putText(fusion_proposal_top, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA)
# concatenate top_view_log and final prediction
top_view_log = np.concatenate((top_view_log, fusion_proposal_top), 1)
top_view_log = np.concatenate((top_view_log, prediction_top), 1)
self.summary_image(top_view_log, scope_name + '/top_view', step=step)
# prediction on rgb
text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)]
prediction_on_rgb = nud.draw_box3d_on_camera(self.batch_rgb_images[0], self.boxes3d,
text_lables=text_lables)
self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
# load
objs = raw_tracklet.load(one_frame_tag)
rgb = raw_img.load(one_frame_tag)
lidar = raw_lidar.load(one_frame_tag)
# preprocess
rgb = preprocess.rgb(rgb)
top = preprocess.lidar_to_top(lidar)
boxes3d = [preprocess.bbox3d(obj) for obj in objs]
labels = [preprocess.label(obj) for obj in objs]
# dump rgb
img = draw.draw_box3d_on_camera(rgb, boxes3d)
new_size = (img.shape[1] // 3, img.shape[0] // 3)
img = cv2.resize(img, new_size)
path = os.path.join(config.cfg.LOG_DIR, 'test', 'rgb',
'%s.png' % one_frame_tag.replace('/', '_'))
cv2.imwrite(path, img)
print('write %s finished' % path)
# draw bbox on top image
path = os.path.join(config.cfg.LOG_DIR, 'test', 'top',
'%s.png' % one_frame_tag.replace('/', '_'))
top_image = data.draw_top_image(top)
top_image = data.draw_box3d_on_top(top_image,
boxes3d,
color=(0, 0, 80))
cv2.imwrite(path, top_image)
print('write %s finished' % path)
def __call__(self, max_iter=1000, train_set =None, validation_set =None):
sess = self.sess
net = self.net
with sess.as_default():
#for init model
batch_size=1
validation_step=40
ckpt_save_step=200
if cfg.TRAINING_TIMER:
time_it = timer()
# start training here #########################################################################################
self.log_msg.write('iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n')
self.log_msg.write('-------------------------------------------------------------------------------------\n')
for iter in range(max_iter):
is_validation = False
summary_it = False
summary_runmeta = False
print_loss = False
log_this_iter = False
# set fit flag
if iter % validation_step == 0: summary_it,is_validation,print_loss = True,True,True # summary validation loss
if (iter+1) % validation_step == 0: summary_it,print_loss = True,True # summary train loss
if iter % 5 == 0: print_loss = True #print train loss
if 1 and iter%300 == 3: summary_it,summary_runmeta = True,True
if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0:
log_this_iter = True
print('summary log image')
if iter % self.iter_debug == 0: is_validation =False
else: is_validation =True
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer
step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load(batch_size, shuffled=True)
#
# if self.batch_data_is_invalid(self.batch_gt_boxes3d[0]):
# continue
# for keeping all gt labels and gt boxes inside range, and discard gt out of selected range.
is_gt_inside_range, batch_gt_labels_in_range, batch_gt_boxes3d_in_range = \
self.keep_gt_inside_range(self.batch_gt_labels[0], self.batch_gt_boxes3d[0])
if not is_gt_inside_range: continue
# todo current support only batch_size == 1
self.batch_gt_labels = np.zeros((1, batch_gt_labels_in_range.shape[0]), dtype=np.int32)
self.batch_gt_boxes3d = np.zeros((1, batch_gt_labels_in_range.shape[0], 8, 3), dtype=np.float32)
self.batch_gt_labels[0] = batch_gt_labels_in_range
self.batch_gt_boxes3d[0] = batch_gt_boxes3d_in_range
# fit_iterate log init
if log_this_iter:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info(self.frame_id)[0]
self.log_subdir = step_name + '/' + self.time_str
top_image = data.draw_top_image(self.batch_top_view[0])
self.top_image = self.top_image_padding(top_image)
# fit
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \
self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view,
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id,
is_validation =is_validation, summary_it=summary_it,
summary_runmeta=summary_runmeta, log=log_this_iter)
if print_loss:
self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \
(step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss))
if iter%ckpt_save_step==0:
# saver.save(sess, pretrained_model_path)
print('save_weights')
self.save_weights(self.train_target)
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \
(time_it.time_diff_per_n_loops(), ckpt_save_step))
self.gc()
self.n_global_step += 1
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \
(time_it.total_time()))
self.save_progress()
def __call__(self, max_iter=1000, train_set=None, validation_set=None):
sess = self.sess
net = self.net
with sess.as_default():
#for init model
batch_size = 1
validation_step = 40
ckpt_save_step = 200
if cfg.TRAINING_TIMER:
time_it = timer()
# start training here #########################################################################################
self.log_msg.write(
'iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n'
)
self.log_msg.write(
'-------------------------------------------------------------------------------------\n'
)
for iter in range(max_iter):
is_validation = False
summary_it = False
summary_runmeta = False
print_loss = False
log_this_iter = False
# set fit flag
if iter % validation_step == 0:
summary_it, is_validation, print_loss = True, True, True # summary validation loss
if (iter + 1) % validation_step == 0:
summary_it, print_loss = True, True # summary train loss
if iter % 5 == 0: print_loss = True #print train loss
if 1 and iter % 300 == 3:
summary_it, summary_runmeta = True, True
if iter % self.iter_debug == 0 or (iter +
1) % self.iter_debug == 0:
log_this_iter = True
print('summary log image')
if iter % self.iter_debug == 0: is_validation = False
else: is_validation = True
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer
step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load(batch_size, shuffled=True)
#
# if self.batch_data_is_invalid(self.batch_gt_boxes3d[0]):
# continue
# for keeping all gt labels and gt boxes inside range, and discard gt out of selected range.
is_gt_inside_range, batch_gt_labels_in_range, batch_gt_boxes3d_in_range = \
self.keep_gt_inside_range(self.batch_gt_labels[0], self.batch_gt_boxes3d[0])
if not is_gt_inside_range: continue
# todo current support only batch_size == 1
self.batch_gt_labels = np.zeros(
(1, batch_gt_labels_in_range.shape[0]), dtype=np.int32)
self.batch_gt_boxes3d = np.zeros(
(1, batch_gt_labels_in_range.shape[0], 8, 3),
dtype=np.float32)
self.batch_gt_labels[0] = batch_gt_labels_in_range
self.batch_gt_boxes3d[0] = batch_gt_boxes3d_in_range
# fit_iterate log init
if log_this_iter:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info(self.frame_id)[0]
self.log_subdir = step_name + '/' + self.time_str
top_image = data.draw_top_image(self.batch_top_view[0])
self.top_image = self.top_image_padding(top_image)
# fit
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \
self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view,
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id,
is_validation =is_validation, summary_it=summary_it,
summary_runmeta=summary_runmeta, log=log_this_iter)
if print_loss:
self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \
(step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss))
if iter % ckpt_save_step == 0:
# saver.save(sess, pretrained_model_path)
print('save_weights')
self.save_weights(self.train_target)
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \
(time_it.time_diff_per_n_loops(), ckpt_save_step))
self.gc()
self.n_global_step += 1
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \
(time_it.total_time()))
self.save_progress()
def fit_iteration(self,
is_validation=False,
summary_it=False,
summary_runmeta=False,
log_image=False,
summary_iou=False,
iou_statistic=False):
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = \
self.val_summary_writer if is_validation else self.train_summary_writer
self.step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, self.calib = \
data_set.load()
# print(self.raw_img.files_path_mapping[self.frame_id])
# resize rgb images
self.batch_rgb_images = np.array([cv2.resize(x, (self.rgb_shape[1], self.rgb_shape[0])) \
for x in self.batch_rgb_images])
# fit_iterate log init
if log_image:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info()
self.log_subdir = self.step_name + '/' + self.time_str
self.top_image = data.draw_top_image(self.batch_top_view[0])
# self.top_image = self.top_image_padding(top_image)
net = self.net
sess = self.sess
# put tensorboard inside
top_cls_loss = net['top_cls_loss']
top_reg_loss = net['top_reg_loss']
fuse_cls_loss = net['fuse_cls_loss']
fuse_reg_loss = net['fuse_reg_loss']
self.batch_gt_top_boxes = box.box3d_to_top_box(
self.batch_gt_boxes3d[0])
## run propsal generation
fd1 = {
net['top_view']: self.batch_top_view,
net['top_anchors']: self.top_view_anchors,
net['top_inside_inds']: self.anchors_inside_inds,
blocks.IS_TRAIN_PHASE: True,
K.learning_phase(): 1
}
self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \
sess.run([net['train_proposals'], net['train_proposal_scores'], net['top_features']], fd1)
## generate train rois for RPN
self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \
rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0],
self.batch_gt_top_boxes)
self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \
rcnn_target(self.batch_proposals, self.batch_gt_labels[0],
self.batch_gt_top_boxes, self.batch_gt_boxes3d[0])
# print(self.anchors_details())
# print(self.rpn_poposal_details())
self.batch_rois3d = project_to_roi3d(self.batch_top_rois)
self.batch_front_rois = project_to_front_roi(self.batch_rois3d)
self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d,
self.calib.velo_to_rgb)
## run classification and regression loss -----------
fd2 = {
**fd1,
net['top_view']: self.batch_top_view,
net['front_view']: self.batch_front_view,
net['rgb_images']: self.batch_rgb_images,
net['top_rois']: self.batch_top_rois,
net['front_rois']: self.batch_front_rois,
net['rgb_rois']: self.batch_rgb_rois,
net['top_inds']: self.batch_top_inds,
net['top_pos_inds']: self.batch_top_pos_inds,
net['top_labels']: self.batch_top_labels,
net['top_targets']: self.batch_top_targets,
net['fuse_labels']: self.batch_fuse_labels,
net['fuse_targets']: self.batch_fuse_targets,
}
"""
if self.debug_mode:
print('\n\nstart debug mode\n\n')
debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess)
debug_sess.add_tensor_filter('has_inf_or_nan', tf_debug.has_inf_or_nan)
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
"""
if summary_it:
run_options = None
run_metadata = None
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2)
self.val_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
# print('added validation summary ')
else:
if summary_runmeta:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss,
self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata)
self.train_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
# print('added training summary ')
if summary_runmeta:
self.train_summary_writer.add_run_metadata(
run_metadata, 'step%d' % self.n_global_step)
# print('added runtime metadata.')
else:
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
else:
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss],
feed_dict=fd2)
if log_image:
# t0 = time.time()
step_name = 'validation' if is_validation else 'train'
scope_name = '%s_iter_%06d' % (
step_name, self.n_global_step -
(self.n_global_step % self.iter_debug))
boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, \
self.batch_rgb_images, self.calib.velo_to_rgb)
if 0 and set(self.train_target) != set(mv3d_net.top_view_rpn_name):
# get iou
iou = -1
inds = np.where(self.batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds],
boxes3d)
if iou_statistic: self.iou_store.append(iou)
if summary_iou:
iou_aver = sum(self.iou_store) / len(self.iou_store)
self.iou_store = []
tag = os.path.join('IOU')
self.summary_scalar(value=iou_aver,
tag=tag,
step=self.n_global_step)
self.log_msg.write('\n %s iou average: %.5f\n' %
(self.step_name, iou_aver))
except ValueError:
# print("waring :", sys.exc_info()[0])
pass
#set scope name
if iou == -1:
scope_name = os.path.join(scope_name,
'iou_error'.format(range(5, 8)))
else:
for iou_range in self.log_iou_range:
if int(iou * 100) in iou_range:
scope_name = os.path.join(
scope_name, 'iou_{}'.format(iou_range))
# print('Summary log image, scope name: {}'.format(scope_name))
self.log_fusion_net_target(self.batch_rgb_images[0],
scope_name=scope_name)
log_info_str = 'frame info: ' + self.frame_info + '\n'
log_info_str += self.anchors_details()
log_info_str += self.rpn_poposal_details()
log_info_str += '\n'
self.log_info(self.log_subdir, log_info_str)
self.predict_log(self.log_subdir,
self.calib.velo_to_rgb,
log_rpn=True,
step=self.n_global_step,
scope_name=scope_name,
loss=(f_cls_loss, f_reg_loss),
frame_tag=self.frame_id,
is_train_mode=True)
# self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0))
return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def pred_and_save(tracklet_pred_dir, dataset, generate_video=False,
frame_offset=16, log_tag=None, weights_tag=None):
# Tracklet_saver will check whether the file already exists.
tracklet = Tracklet_saver(tracklet_pred_dir)
os.makedirs (os.path.join(log_dir,'image'),exist_ok=True)
top_shape, front_shape, rgb_shape=dataset.get_shape()
predict=mv3d.Predictor(top_shape, front_shape, rgb_shape, log_tag=log_tag, weights_tag=weights_tag)
if generate_video:
vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir,'output.mp4'))
# timer
timer_step = 100
if cfg.TRACKING_TIMER:
time_it = timer()
frame_num = 0
for i in range(dataset.size):
rgb, top, front, _, _,_= dataset.load(1)
frame_num = i - frame_offset
if frame_num < 0:
continue
boxes3d,probs=predict(top, front, rgb)
predict.dump_log(log_subdir=log_subdir,n_frame=i)
# time timer_step iterations. Turn it on/off in config.py
if cfg.TRACKING_TIMER and i%timer_step ==0 and i!=0:
predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \
(time_it.time_diff_per_n_loops(), timer_step))
# for debugging: save image and show image.
top_image = draw_top_image(top[0])
rgb_image = rgb[0]
if len(boxes3d)!=0:
top_image = draw_box3d_on_top(top_image, boxes3d[:,:,:], color=(80, 80, 0), thickness=3)
rgb_image = draw_box3d_on_camera(rgb_image, boxes3d[:, :, :], color=(0, 0, 80), thickness=3)
translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :])
#todo: remove it after gtbox is ok
size[:,1:3] = size[:,1:3]/cfg.TRACKLET_GTBOX_LENGTH_SCALE
for j in range(len(translation)):
if 0<translation[j,1]<8:
tracklet.add_tracklet(frame_num, size[j], translation[j], rotation[j])
resize_scale=top_image.shape[0]/rgb_image.shape[0]
rgb_image = cv2.resize(rgb_image,(int(rgb_image.shape[1]*resize_scale), top_image.shape[0]))
rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB)
new_image = np.concatenate((top_image, rgb_image), axis = 1)
cv2.imwrite(os.path.join(log_dir,'image','%5d_image.jpg'%i), new_image)
if generate_video:
vid_in.writeFrame(new_image)
vid_in.close()
tracklet.write_tracklet()
predict.dump_weigths(os.path.join(log_dir, 'pretrained_model'))
if cfg.TRACKING_TIMER:
predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \
(time_it.total_time()))
print("tracklet file named tracklet_labels.xml is written successfully.")
return tracklet.path
def iteration_summary(self, prefix, summaryWriter):
"""
Method for creating numerous image summaries.
:param prefix: Can be 'training' or 'validation'
:param summaryWriter: Writer instance either for training or validation
"""
# get the indices of pos+neg, positive anchors as well as labels and regression targets
top_inds, pos_inds, labels, targets = self.network.get_targets(
self.batch_gt_labels, self.batch_gt_boxes3d)
top = data.draw_top_image(self.batch_top_view[0]).copy()
self.log_image(step=self.n_global_step,
prefix=prefix,
frame_tag=self.frame_id,
summary_writer=summaryWriter)
proposals, scores, probs = self.predict(self.batch_top_view,
self.batch_rgb_images)
nms_proposals, nms_scores = self.post_process(proposals,
probs,
minProb=0.1)
######################
## probabilities ##
######################
for i in range(1, cfg.NUM_CLASSES):
# reshape from [-1, NUM_CLASSES]
# @TODO don't make the reshape fixed
self.summary_image(np.reshape(probs[::2, i], (176 * 2, 200 * 2)),
prefix + '/probabilities' + str(i) + '_Up',
summaryWriter,
step=self.n_global_step)
self.summary_image(np.reshape(probs[1::2, i], (176 * 2, 200 * 2)),
prefix + '/probabilities' + str(i) + '_Right',
summaryWriter,
step=self.n_global_step)
######################
## proposals ##
######################
proposalIdx = np.where(np.argmax(probs, axis=1) != 0)
numProposals = np.sum(proposalIdx)
Boxes3d = []
if (numProposals > 0):
Boxes3d = net.processing.boxes.box_transform_voxelnet_inv(
proposals[proposalIdx], self.network.anchors[proposalIdx])
topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d)
self.summary_image(topbox,
prefix + '/proposalBoxes',
summaryWriter,
step=self.n_global_step)
######################
## top 10 proposals ##
######################
sortedIdx = np.argsort(-probs[:, 1].flatten())
top10Proposals = proposals[sortedIdx[:10]]
### top visualization ###
Boxes3d = net.processing.boxes.box_transform_voxelnet_inv(
top10Proposals, self.network.anchors[sortedIdx[:10]])
topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d)
self.summary_image(topbox,
prefix + '/top10proposalBoxes',
summaryWriter,
step=self.n_global_step)
#######################
## nms visualization ##
#######################
topbox = top
for i in range(0, cfg.NUM_CLASSES - 1):
topbox = net.processing.boxes3d.draw_box3d_on_top(
topbox, nms_proposals[i][:10, :, :], scores=nms_scores[i])
self.summary_image(topbox,
prefix + '/top10NMSBoxes',
summaryWriter,
step=self.n_global_step)
### to rgb image ###
colors = [(255, 0, 255), (255, 0, 0), (0, 255, 0), (0, 0, 255)]
rgb_image = self.batch_rgb_images[0]
for i in range(0, cfg.NUM_CLASSES - 1):
text_labels = ['Prob: %.4f' % (prob) for prob in nms_scores[i]]
rgb_image = nud.draw_box3d_on_camera(rgb_image,
nms_proposals[i][:, :, :],
text_lables=text_labels,
color=colors[i])
# add the groundtruth
# Boxes3d = net.processing.boxes.box_transform_voxelnet_inv(targets, self.voxelnet.anchors[pos_inds])
# rgb_image = nud.draw_box3d_on_camera(rgb_image, Boxes3d,
# text_lables=text_labels, color=(0, 255, 255))
self.summary_image(rgb_image,
prefix + '/prediction_on_rgb',
summaryWriter,
step=self.n_global_step)
######################
## groundtruth ##
######################
Boxes3d = net.processing.boxes.box_transform_voxelnet_inv(
targets, self.network.anchors[pos_inds])
topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d)
self.summary_image(topbox,
prefix + '/Targets',
summaryWriter,
step=self.n_global_step)
def __call__(self, max_iter=1000, train_set=None, validation_set=None):
sess = self.sess
net = self.net
with sess.as_default():
#for init model
batch_size = 1
validation_step = 40
ckpt_save_step = 200
if cfg.TRAINING_TIMER:
time_it = timer()
# start training here #########################################################################################
self.log_msg.write(
'iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n'
)
self.log_msg.write(
'-------------------------------------------------------------------------------------\n'
)
for iter in range(max_iter):
is_validation = False
summary_it = False
summary_runmeta = False
print_loss = False
log_this_iter = False
# set fit flag
if iter % validation_step == 0:
summary_it, is_validation, print_loss = True, True, True # summary validation loss
if (iter + 1) % validation_step == 0:
summary_it, print_loss = True, True # summary train loss
if iter % 20 == 0: print_loss = True #print train loss
if 1 and iter % 300 == 0:
summary_it, summary_runmeta = True, True
if iter % self.iter_debug == 0 or (iter +
1) % self.iter_debug == 0:
log_this_iter = True
print('Summary log image')
if iter % self.iter_debug == 0: is_validation = False
else: is_validation = True
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer
step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load()
# fit_iterate log init
if log_this_iter:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info()
self.log_subdir = step_name + '/' + self.time_str
top_image = data.draw_top_image(self.batch_top_view[0])
self.top_image = self.top_image_padding(top_image)
# fit
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \
self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view,
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id,
is_validation =is_validation, summary_it=summary_it,
summary_runmeta=summary_runmeta, log=log_this_iter)
if print_loss:
self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \
(step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss))
if iter % ckpt_save_step == 0:
self.save_weights(self.train_target)
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \
(time_it.time_diff_per_n_loops(), ckpt_save_step))
self.gc()
self.n_global_step += 1
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \
(time_it.total_time()))
self.save_progress()
def fit_iteration(self, is_validation =False, summary_it=False, summary_runmeta=False, log_image=False,
summary_iou=False, iou_statistic=False):
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = \
self.val_summary_writer if is_validation else self.train_summary_writer
self.step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load()
# fit_iterate log init
if log_image:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info()
self.log_subdir = self.step_name + '/' + self.time_str
self.top_image = data.draw_top_image(self.batch_top_view[0])
# self.top_image = self.top_image_padding(top_image)
net = self.net
sess = self.sess
# put tensorboard inside
top_cls_loss = net['top_cls_loss']
top_reg_loss = net['top_reg_loss']
fuse_cls_loss = net['fuse_cls_loss']
fuse_reg_loss = net['fuse_reg_loss']
self.batch_gt_top_boxes = data.box3d_to_top_box(self.batch_gt_boxes3d[0])
## run propsal generation
fd1 = {
net['top_view']: self.batch_top_view,
net['top_anchors']: self.top_view_anchors,
net['top_inside_inds']: self.anchors_inside_inds,
blocks.IS_TRAIN_PHASE: True,
K.learning_phase(): 1
}
self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \
sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1)
## generate train rois for RPN
self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \
rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0],
self.batch_gt_top_boxes)
self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \
fusion_target(self.batch_proposals, self.batch_gt_labels[0],
self.batch_gt_top_boxes, self.batch_gt_boxes3d[0])
self.batch_rois3d = project_to_roi3d(self.batch_top_rois)
self.batch_front_rois = project_to_front_roi(self.batch_rois3d)
self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d)
## run classification and regression loss -----------
fd2 = {
**fd1,
net['top_view']: self.batch_top_view,
net['front_view']: self.batch_front_view,
net['rgb_images']: self.batch_rgb_images,
net['top_rois']: self.batch_top_rois,
net['front_rois']: self.batch_front_rois,
net['rgb_rois']: self.batch_rgb_rois,
net['top_inds']: self.batch_top_inds,
net['top_pos_inds']: self.batch_top_pos_inds,
net['top_labels']: self.batch_top_labels,
net['top_targets']: self.batch_top_targets,
net['fuse_labels']: self.batch_fuse_labels,
net['fuse_targets']: self.batch_fuse_targets,
}
if self.debug_mode:
print('\n\nstart debug mode\n\n')
debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess)
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
if summary_it:
run_options = None
run_metadata = None
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2)
self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step)
# print('added validation summary ')
else:
if summary_runmeta:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss,
self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata)
self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step)
# print('added training summary ')
if summary_runmeta:
self.train_summary_writer.add_run_metadata(run_metadata, 'step%d' % self.n_global_step)
# print('added runtime metadata.')
else:
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
else:
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss],
feed_dict=fd2)
if log_image:
# t0 = time.time()
step_name = 'validation' if is_validation else 'train'
scope_name = '%s_iter_%06d' % (step_name, self.n_global_step - (self.n_global_step % self.iter_debug))
boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, self.batch_rgb_images)
# get iou
iou = -1
inds = np.where(self.batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d)
if iou_statistic: self.iou_store.append(iou)
if summary_iou:
iou_aver = sum(self.iou_store)/len(self.iou_store)
self.iou_store=[]
tag = os.path.join('IOU')
self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step)
self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver))
except ValueError:
# print("waring :", sys.exc_info()[0])
pass
#set scope name
if iou == -1:
scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8)))
else:
for iou_range in self.log_iou_range:
if int(iou*100) in iou_range:
scope_name = os.path.join(scope_name , 'iou_{}'.format (iou_range))
# print('Summary log image, scope name: {}'.format(scope_name))
self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name)
log_info_str = 'frame info: ' + self.frame_info + '\n'
log_info_str += self.anchors_details()
log_info_str += self.rpn_poposal_details()
self.log_info(self.log_subdir, log_info_str)
self.predict_log(self.log_subdir, log_rpn=True, step=self.n_global_step,
scope_name=scope_name, loss=(f_cls_loss, f_reg_loss),
frame_tag=self.frame_id, is_train_mode=True)
# self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0))
return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def pred_and_save(tracklet_pred_dir,
dataset,
generate_video=False,
frame_offset=16,
log_tag=None,
weights_tag=None):
# Tracklet_saver will check whether the file already exists.
tracklet = Tracklet_saver(tracklet_pred_dir, 'pred')
os.makedirs(os.path.join(log_dir, 'image'), exist_ok=True)
gt_tracklet = Tracklet_saver(tracklet_pred_dir, 'gt')
top_shape, front_shape, rgb_shape = dataset.get_shape()
predict = mv3d.Predictor(top_shape,
front_shape,
rgb_shape,
log_tag=log_tag,
weights_tag=weights_tag)
if generate_video:
vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir, 'output.mp4'))
# timer
timer_step = 100
if cfg.TRACKING_TIMER:
time_it = timer()
print('dataset.size')
print(dataset.size)
lenght = []
gt_lenght = []
frame_num = 0
for i in range(dataset.size if fast_test == False else frame_offset + 1):
rgb, top, front, _, _, _ = dataset.load(size=1)
frame_num = i - frame_offset
print('frame_num')
print(frame_num)
if frame_num < 0:
continue
gt_boxes3d_tmp = np.load(
'/home/mohsen/Desktop/MV3D/data/preprocessed/kitti/gt_boxes3d/object3d/test/%05d.npy'
% i)
#remove gt boxes with hiegh less than 40
gt_boxes3d_list = []
for gt_box3d_tmp in gt_boxes3d_tmp:
# if gt_box3d_tmp[0,0]>0:
gt_box3d_tmp_list = []
gt_box3d_tmp_list.append(gt_box3d_tmp)
gt_project = box3d.box3d_to_rgb_box(gt_box3d_tmp_list)
if abs(gt_project[0][0, 1] - gt_project[0][4, 1]) >= 40:
gt_box3d = gt_box3d_tmp
gt_boxes3d_list.append(gt_box3d)
gt_boxes3d = np.array(gt_boxes3d_list)
# gt_boxes3d = gt_boxes3d_tmp
#####################################
boxes3d_tmp, probs = predict(top, front, rgb)
predict.dump_log(log_subdir=log_subdir, n_frame=i)
# time timer_step iterations. Turn it on/off in config.py
if cfg.TRACKING_TIMER and i % timer_step == 0 and i != 0:
predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \
(time_it.time_diff_per_n_loops(), timer_step))
# for debugging: save image and show image.
top_image = draw_top_image(top[0])
rgb_image = rgb[0]
if len(gt_boxes3d) != 0:
gt_lenght.append(len(gt_boxes3d))
gt_translation, gt_size, gt_rotation = boxes3d_decompose(
gt_boxes3d[:, :, :])
# todo: remove it after gtbox is ok
gt_size[:, 1:3] = gt_size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE
for j in range(len(gt_translation)):
gt_tracklet.add_tracklet(frame_num, gt_size[j],
gt_translation[j], gt_rotation[j])
#remove predicted boxes with hiegh less than 40
boxes3d_list = []
for box3d_tmp in boxes3d_tmp:
# if box3d_tmp[0, 0] > 0:
box3d_tmp_list = []
box3d_tmp_list.append(box3d_tmp)
project = box3d.box3d_to_rgb_box(box3d_tmp_list)
if abs(project[0][0, 1] - project[0][4, 1]) >= 40:
print(project[0][0, 1] - project[0][4, 1])
pred_box3d = box3d_tmp
boxes3d_list.append(pred_box3d)
boxes3d = np.array(boxes3d_list)
# boxes3d = boxes3d_tmp
#####################################
print('sizes')
print(np.size(boxes3d))
print(gt_boxes3d)
print(np.size(gt_boxes3d))
if len(boxes3d) != 0:
lenght.append(len(boxes3d))
top_image = draw_box3d_on_top(top_image,
boxes3d[:, :, :],
color=(80, 80, 0),
thickness=3)
rgb_image = draw_box3d_on_camera(rgb_image,
boxes3d[:, :, :],
color=(0, 0, 80),
thickness=3)
if len(gt_boxes3d) != 0:
rgb_image = draw_box3d_on_camera(rgb_image,
gt_boxes3d[:, :, :],
color=(0, 80, 0),
thickness=3)
translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :])
# todo: remove it after gtbox is ok
size[:, 1:3] = size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE
for j in range(len(translation)):
tracklet.add_tracklet(frame_num, size[j], translation[j],
rotation[j])
resize_scale = top_image.shape[0] / rgb_image.shape[0]
rgb_image = cv2.resize(
rgb_image,
(int(rgb_image.shape[1] * resize_scale), top_image.shape[0]))
rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB)
new_image = np.concatenate((top_image, rgb_image), axis=1)
cv2.imwrite(os.path.join(log_dir, 'image', '%5d_image.jpg' % i),
new_image)
if generate_video:
vid_in.writeFrame(new_image)
vid_in.close()
print(lenght)
print(sum(lenght))
tracklet.write_tracklet()
predict.dump_weigths(os.path.join(log_dir, 'pretrained_model'))
print(gt_lenght)
print(sum(gt_lenght))
gt_tracklet.write_tracklet()
if cfg.TRACKING_TIMER:
predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \
(time_it.total_time()))
print("tracklet file named tracklet_labels.xml is written successfully.")
return tracklet.path, gt_tracklet.path
# num libs
import math
import random
import numpy as np
import cv2
import config
import data
import net.utility.draw as draw
if __name__ == '__main__':
# preprocessed_dir=config.cfg.PREPROCESSED_DATA_SETS_DIR
preprocessed_dir=config.cfg.PREPROCESSING_DATA_SETS_DIR
dataset='/1/15/00070.npy'
top_view_dir =preprocessed_dir+ '/top'+dataset
gt_boxes3d_dir =preprocessed_dir+'/gt_boxes3d'+dataset
top=np.load(top_view_dir)
gt_boxes3d=np.load(gt_boxes3d_dir)
top_img=data.draw_top_image(top)
top_img_marked=data.draw_box3d_on_top(top_img,gt_boxes3d)
draw.imsave('top_img_marked',top_img_marked,'debug')
print('top_img_marked dump finished!!')
def pred_and_save(tracklet_pred_dir,
dataset,
generate_video=False,
frame_offset=16,
log_tag=None,
weights_tag=None):
# Tracklet_saver will check whether the file already exists.
tracklet = Tracklet_saver(tracklet_pred_dir)
os.makedirs(os.path.join(log_dir, 'image'), exist_ok=True)
top_shape, front_shape, rgb_shape = dataset.get_shape()
predict = mv3d.Predictor(top_shape,
front_shape,
rgb_shape,
log_tag=log_tag,
weights_tag=weights_tag)
if generate_video:
vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir, 'output.mp4'))
# timer
timer_step = 100
if cfg.TRACKING_TIMER:
time_it = timer()
frame_num = 0
for i in range(dataset.size if fast_test == False else frame_offset + 1):
rgb, top, front, _, _, _ = dataset.load()
frame_num = i - frame_offset
if frame_num < 0:
continue
boxes3d, probs = predict(top, front, rgb)
predict.dump_log(log_subdir=log_subdir, n_frame=i)
# time timer_step iterations. Turn it on/off in config.py
if cfg.TRACKING_TIMER and i % timer_step == 0 and i != 0:
predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \
(time_it.time_diff_per_n_loops(), timer_step))
# for debugging: save image and show image.
top_image = draw_top_image(top[0])
rgb_image = rgb[0]
if len(boxes3d) != 0:
top_image = draw_box3d_on_top(top_image,
boxes3d[:, :, :],
color=(80, 80, 0),
thickness=3)
rgb_image = draw_box3d_on_camera(rgb_image,
boxes3d[:, :, :],
color=(0, 0, 80),
thickness=3)
translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :])
# todo: remove it after gtbox is ok
size[:, 1:3] = size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE
for j in range(len(translation)):
if 0 < translation[j, 1] < 8:
tracklet.add_tracklet(frame_num, size[j], translation[j],
rotation[j])
resize_scale = top_image.shape[0] / rgb_image.shape[0]
rgb_image = cv2.resize(
rgb_image,
(int(rgb_image.shape[1] * resize_scale), top_image.shape[0]))
rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB)
new_image = np.concatenate((top_image, rgb_image), axis=1)
cv2.imwrite(os.path.join(log_dir, 'image', '%5d_image.jpg' % i),
new_image)
if generate_video:
vid_in.writeFrame(new_image)
vid_in.close()
tracklet.write_tracklet()
predict.dump_weigths(os.path.join(log_dir, 'pretrained_model'))
if cfg.TRACKING_TIMER:
predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \
(time_it.total_time()))
print("tracklet file named tracklet_labels.xml is written successfully.")
return tracklet.path
def draw_bbox_on_lidar_top(top, boxes3d, one_frame_tag):
path = os.path.join(config.cfg.LOG_DIR, 'test', 'top', '%s.png' % one_frame_tag.replace('/', '_'))
top_image = data.draw_top_image(top)
top_image = data.draw_box3d_on_top(top_image, boxes3d, color=(0, 0, 80))
cv2.imwrite(path, top_image)
print('write %s finished' % path)
def __call__(self, max_iter=1000, train_set =None, validation_set =None):
sess = self.sess
net = self.net
with sess.as_default():
#for init model
batch_size=1
validation_step=40
ckpt_save_step=200
if cfg.TRAINING_TIMER:
time_it = timer()
# start training here #########################################################################################
self.log_msg.write('iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n')
self.log_msg.write('-------------------------------------------------------------------------------------\n')
for iter in range(max_iter):
is_validation = False
summary_it = False
summary_runmeta = False
print_loss = False
log_this_iter = False
# set fit flag
if iter % validation_step == 0: summary_it,is_validation,print_loss = True,True,True # summary validation loss
if (iter+1) % validation_step == 0: summary_it,print_loss = True,True # summary train loss
if iter % 20 == 0: print_loss = True #print train loss
if 1 and iter%300 == 0: summary_it,summary_runmeta = True,True
if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0:
log_this_iter = True
print('Summary log image')
if iter % self.iter_debug == 0: is_validation =False
else: is_validation =True
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer
step_name = 'validation' if is_validation else 'training'
# load dataset
if config.cfg.USE_RNN:
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load(batch_size, shuffled=False)
else:
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load(batch_size, shuffled=True)
# fit_iterate log init
if log_this_iter:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info(self.frame_id)[0]
self.log_subdir = step_name + '/' + self.time_str
top_image = data.draw_top_image(self.batch_top_view[0])
self.top_image = self.top_image_padding(top_image)
# fit
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \
self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view,
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id,
is_validation =is_validation, summary_it=summary_it,
summary_runmeta=summary_runmeta, log=log_this_iter)
if print_loss:
self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \
(step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss))
if iter%ckpt_save_step==0:
self.save_weights(self.train_target)
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \
(time_it.time_diff_per_n_loops(), ckpt_save_step))
self.gc()
self.n_global_step += 1
if cfg.TRAINING_TIMER:
self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \
(time_it.total_time()))
self.save_progress()
