def log_prediction(self,
batch_top_view,
batch_front_view,
batch_rgb_images,
batch_gt_labels=None,
batch_gt_boxes3d=None,
print_iou=False,
log_rpn=False,
step=None,
scope_name=''):
boxes3d, lables = self.predict(batch_top_view, batch_front_view,
batch_rgb_images)
self.predict_log(self.log_subdir,
log_rpn=log_rpn,
step=step,
scope_name=scope_name)
if type(batch_gt_boxes3d) == np.ndarray and type(
batch_gt_labels) == np.ndarray:
inds = np.where(batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds], boxes3d)
tag = os.path.join(scope_name, 'IOU')
self.summary_scalar(value=iou,
tag=tag,
step=self.n_global_step)
except ValueError:
iou = -1
print("waring :", sys.exc_info()[0])
if print_iou:
self.log_msg.write('\n %s iou: %.5f\n' % (scope_name, iou))
def log_prediction(self, batch_top_view, batch_front_view, batch_rgb_images,
batch_gt_labels=None, batch_gt_boxes3d=None, print_iou=False,
log_rpn=False, step=None, scope_name='',loss:tuple =None):
boxes3d, lables = self.predict(batch_top_view, batch_front_view, batch_rgb_images)
self.predict_log(self.log_subdir,log_rpn=log_rpn, step=step, scope_name=scope_name, loss=loss,
frame_tag=self.frame_id, is_train_mode=True)
if type(batch_gt_boxes3d)==np.ndarray and type(batch_gt_labels)==np.ndarray:
inds = np.where(batch_gt_labels[0]!=0)
try:
iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds], boxes3d)
tag = os.path.join('IOU')
self.summary_scalar(value=iou, tag=tag, step=self.n_global_step)
except ValueError:
iou= -1
print("waring :", sys.exc_info()[0])
if print_iou: self.log_msg.write('\n %s iou: %.5f\n' % (self.step_name, iou))
def fit_iteration(self,
is_validation=False,
summary_it=False,
summary_runmeta=False,
log_image=False,
summary_iou=False,
iou_statistic=False):
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = \
self.val_summary_writer if is_validation else self.train_summary_writer
self.step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, self.calib = \
data_set.load()
# print(self.raw_img.files_path_mapping[self.frame_id])
# resize rgb images
self.batch_rgb_images = np.array([cv2.resize(x, (self.rgb_shape[1], self.rgb_shape[0])) \
for x in self.batch_rgb_images])
# fit_iterate log init
if log_image:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info()
self.log_subdir = self.step_name + '/' + self.time_str
self.top_image = data.draw_top_image(self.batch_top_view[0])
# self.top_image = self.top_image_padding(top_image)
net = self.net
sess = self.sess
# put tensorboard inside
top_cls_loss = net['top_cls_loss']
top_reg_loss = net['top_reg_loss']
fuse_cls_loss = net['fuse_cls_loss']
fuse_reg_loss = net['fuse_reg_loss']
self.batch_gt_top_boxes = box.box3d_to_top_box(
self.batch_gt_boxes3d[0])
## run propsal generation
fd1 = {
net['top_view']: self.batch_top_view,
net['top_anchors']: self.top_view_anchors,
net['top_inside_inds']: self.anchors_inside_inds,
blocks.IS_TRAIN_PHASE: True,
K.learning_phase(): 1
}
self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \
sess.run([net['train_proposals'], net['train_proposal_scores'], net['top_features']], fd1)
## generate train rois for RPN
self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \
rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0],
self.batch_gt_top_boxes)
self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \
rcnn_target(self.batch_proposals, self.batch_gt_labels[0],
self.batch_gt_top_boxes, self.batch_gt_boxes3d[0])
# print(self.anchors_details())
# print(self.rpn_poposal_details())
self.batch_rois3d = project_to_roi3d(self.batch_top_rois)
self.batch_front_rois = project_to_front_roi(self.batch_rois3d)
self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d,
self.calib.velo_to_rgb)
## run classification and regression loss -----------
fd2 = {
**fd1,
net['top_view']: self.batch_top_view,
net['front_view']: self.batch_front_view,
net['rgb_images']: self.batch_rgb_images,
net['top_rois']: self.batch_top_rois,
net['front_rois']: self.batch_front_rois,
net['rgb_rois']: self.batch_rgb_rois,
net['top_inds']: self.batch_top_inds,
net['top_pos_inds']: self.batch_top_pos_inds,
net['top_labels']: self.batch_top_labels,
net['top_targets']: self.batch_top_targets,
net['fuse_labels']: self.batch_fuse_labels,
net['fuse_targets']: self.batch_fuse_targets,
}
"""
if self.debug_mode:
print('\n\nstart debug mode\n\n')
debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess)
debug_sess.add_tensor_filter('has_inf_or_nan', tf_debug.has_inf_or_nan)
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
"""
if summary_it:
run_options = None
run_metadata = None
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2)
self.val_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
# print('added validation summary ')
else:
if summary_runmeta:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss,
self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata)
self.train_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
# print('added training summary ')
if summary_runmeta:
self.train_summary_writer.add_run_metadata(
run_metadata, 'step%d' % self.n_global_step)
# print('added runtime metadata.')
else:
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
else:
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss],
feed_dict=fd2)
if log_image:
# t0 = time.time()
step_name = 'validation' if is_validation else 'train'
scope_name = '%s_iter_%06d' % (
step_name, self.n_global_step -
(self.n_global_step % self.iter_debug))
boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, \
self.batch_rgb_images, self.calib.velo_to_rgb)
if 0 and set(self.train_target) != set(mv3d_net.top_view_rpn_name):
# get iou
iou = -1
inds = np.where(self.batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds],
boxes3d)
if iou_statistic: self.iou_store.append(iou)
if summary_iou:
iou_aver = sum(self.iou_store) / len(self.iou_store)
self.iou_store = []
tag = os.path.join('IOU')
self.summary_scalar(value=iou_aver,
tag=tag,
step=self.n_global_step)
self.log_msg.write('\n %s iou average: %.5f\n' %
(self.step_name, iou_aver))
except ValueError:
# print("waring :", sys.exc_info()[0])
pass
#set scope name
if iou == -1:
scope_name = os.path.join(scope_name,
'iou_error'.format(range(5, 8)))
else:
for iou_range in self.log_iou_range:
if int(iou * 100) in iou_range:
scope_name = os.path.join(
scope_name, 'iou_{}'.format(iou_range))
# print('Summary log image, scope name: {}'.format(scope_name))
self.log_fusion_net_target(self.batch_rgb_images[0],
scope_name=scope_name)
log_info_str = 'frame info: ' + self.frame_info + '\n'
log_info_str += self.anchors_details()
log_info_str += self.rpn_poposal_details()
log_info_str += '\n'
self.log_info(self.log_subdir, log_info_str)
self.predict_log(self.log_subdir,
self.calib.velo_to_rgb,
log_rpn=True,
step=self.n_global_step,
scope_name=scope_name,
loss=(f_cls_loss, f_reg_loss),
frame_tag=self.frame_id,
is_train_mode=True)
# self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0))
return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def fit_iteration(self, is_validation =False, summary_it=False, summary_runmeta=False, log_image=False,
summary_iou=False, iou_statistic=False):
data_set = self.validation_set if is_validation else self.train_set
self.default_summary_writer = \
self.val_summary_writer if is_validation else self.train_summary_writer
self.step_name = 'validation' if is_validation else 'training'
# load dataset
self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \
self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \
data_set.load()
# fit_iterate log init
if log_image:
self.time_str = strftime("%Y_%m_%d_%H_%M", localtime())
self.frame_info = data_set.get_frame_info()
self.log_subdir = self.step_name + '/' + self.time_str
self.top_image = data.draw_top_image(self.batch_top_view[0])
# self.top_image = self.top_image_padding(top_image)
net = self.net
sess = self.sess
# put tensorboard inside
top_cls_loss = net['top_cls_loss']
top_reg_loss = net['top_reg_loss']
fuse_cls_loss = net['fuse_cls_loss']
fuse_reg_loss = net['fuse_reg_loss']
self.batch_gt_top_boxes = data.box3d_to_top_box(self.batch_gt_boxes3d[0])
## run propsal generation
fd1 = {
net['top_view']: self.batch_top_view,
net['top_anchors']: self.top_view_anchors,
net['top_inside_inds']: self.anchors_inside_inds,
blocks.IS_TRAIN_PHASE: True,
K.learning_phase(): 1
}
self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \
sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1)
## generate train rois for RPN
self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \
rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0],
self.batch_gt_top_boxes)
self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \
fusion_target(self.batch_proposals, self.batch_gt_labels[0],
self.batch_gt_top_boxes, self.batch_gt_boxes3d[0])
self.batch_rois3d = project_to_roi3d(self.batch_top_rois)
self.batch_front_rois = project_to_front_roi(self.batch_rois3d)
self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d)
## run classification and regression loss -----------
fd2 = {
**fd1,
net['top_view']: self.batch_top_view,
net['front_view']: self.batch_front_view,
net['rgb_images']: self.batch_rgb_images,
net['top_rois']: self.batch_top_rois,
net['front_rois']: self.batch_front_rois,
net['rgb_rois']: self.batch_rgb_rois,
net['top_inds']: self.batch_top_inds,
net['top_pos_inds']: self.batch_top_pos_inds,
net['top_labels']: self.batch_top_labels,
net['top_targets']: self.batch_top_targets,
net['fuse_labels']: self.batch_fuse_labels,
net['fuse_targets']: self.batch_fuse_targets,
}
if self.debug_mode:
print('\n\nstart debug mode\n\n')
debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess)
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
if summary_it:
run_options = None
run_metadata = None
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2)
self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step)
# print('added validation summary ')
else:
if summary_runmeta:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss,
self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata)
self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step)
# print('added training summary ')
if summary_runmeta:
self.train_summary_writer.add_run_metadata(run_metadata, 'step%d' % self.n_global_step)
# print('added runtime metadata.')
else:
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
else:
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss],
feed_dict=fd2)
if log_image:
# t0 = time.time()
step_name = 'validation' if is_validation else 'train'
scope_name = '%s_iter_%06d' % (step_name, self.n_global_step - (self.n_global_step % self.iter_debug))
boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, self.batch_rgb_images)
# get iou
iou = -1
inds = np.where(self.batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d)
if iou_statistic: self.iou_store.append(iou)
if summary_iou:
iou_aver = sum(self.iou_store)/len(self.iou_store)
self.iou_store=[]
tag = os.path.join('IOU')
self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step)
self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver))
except ValueError:
# print("waring :", sys.exc_info()[0])
pass
#set scope name
if iou == -1:
scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8)))
else:
for iou_range in self.log_iou_range:
if int(iou*100) in iou_range:
scope_name = os.path.join(scope_name , 'iou_{}'.format (iou_range))
# print('Summary log image, scope name: {}'.format(scope_name))
self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name)
log_info_str = 'frame info: ' + self.frame_info + '\n'
log_info_str += self.anchors_details()
log_info_str += self.rpn_poposal_details()
self.log_info(self.log_subdir, log_info_str)
self.predict_log(self.log_subdir, log_rpn=True, step=self.n_global_step,
scope_name=scope_name, loss=(f_cls_loss, f_reg_loss),
frame_tag=self.frame_id, is_train_mode=True)
# self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0))
return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def fit_iteration(self,
batch_rgb_images,
batch_top_view,
batch_front_view,
batch_gt_labels,
batch_gt_boxes3d,
frame_id,
is_validation=False,
summary_it=False,
summary_runmeta=False,
log=False):
net = self.net
sess = self.sess
# put tensorboard inside
top_cls_loss = net['top_cls_loss']
top_reg_loss = net['top_reg_loss']
fuse_cls_loss = net['fuse_cls_loss']
fuse_reg_loss = net['fuse_reg_loss']
self.batch_gt_top_boxes = data.box3d_to_top_box(batch_gt_boxes3d[0])
## run propsal generation
fd1 = {
net['top_view']: batch_top_view,
net['top_anchors']: self.top_view_anchors,
net['top_inside_inds']: self.anchors_inside_inds,
blocks.IS_TRAIN_PHASE: True,
K.learning_phase(): 1
}
self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \
sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1)
## generate train rois for RPN
self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \
rpn_target(self.top_view_anchors, self.anchors_inside_inds, batch_gt_labels[0],
self.batch_gt_top_boxes)
self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \
fusion_target(self.batch_proposals, batch_gt_labels[0], self.batch_gt_top_boxes, batch_gt_boxes3d[0])
self.batch_rois3d = project_to_roi3d(self.batch_top_rois)
self.batch_front_rois = project_to_front_roi(self.batch_rois3d)
self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d)
## run classification and regression loss -----------
fd2 = {
**fd1,
net['top_view']: batch_top_view,
net['front_view']: batch_front_view,
net['rgb_images']: batch_rgb_images,
net['top_rois']: self.batch_top_rois,
net['front_rois']: self.batch_front_rois,
net['rgb_rois']: self.batch_rgb_rois,
net['top_inds']: self.batch_top_inds,
net['top_pos_inds']: self.batch_top_pos_inds,
net['top_labels']: self.batch_top_labels,
net['top_targets']: self.batch_top_targets,
net['fuse_labels']: self.batch_fuse_labels,
net['fuse_targets']: self.batch_fuse_targets,
}
if self.debug_mode:
print('\n\nstart debug mode\n\n')
debug_sess = tf_debug.LocalCLIDebugWrapperSession(sess)
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
if summary_it:
run_options = None
run_metadata = None
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2)
self.val_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
print('added validation summary ')
else:
if summary_runmeta:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss,
self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata)
self.train_summary_writer.add_summary(tb_sum_val,
self.n_global_step)
print('added training summary ')
if summary_runmeta:
self.train_summary_writer.add_run_metadata(
run_metadata, 'step%d' % self.n_global_step)
print('added runtime metadata.')
else:
if is_validation:
t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2)
else:
_, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \
sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss],
feed_dict=fd2)
if log:
t0 = time.time()
step_name = 'validation' if is_validation else 'train'
scope_name = '%s_iter_%06d' % (
step_name, self.n_global_step -
(self.n_global_step % self.iter_debug))
boxes3d, lables = self.predict(batch_top_view, batch_front_view,
batch_rgb_images)
# get iou
iou = -1
if type(batch_gt_boxes3d) == np.ndarray and type(
batch_gt_labels) == np.ndarray:
inds = np.where(batch_gt_labels[0] != 0)
try:
iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds],
boxes3d)
tag = os.path.join('IOU')
self.summary_scalar(value=iou,
tag=tag,
step=self.n_global_step)
except ValueError:
print("waring :", sys.exc_info()[0])
self.log_msg.write('\n %s iou: %.5f\n' % (self.step_name, iou))
#set scope name
if iou == -1:
scope_name = os.path.join(scope_name,
'iou_error'.format(range(5, 8)))
else:
for iou_range in self.log_iou_range:
if int(iou * 100) in iou_range:
scope_name = os.path.join(scope_name,
'iou_{}'.format(iou_range))
print('Summary log image, scope name: {}'.format(scope_name))
self.log_fusion_net_target(batch_rgb_images[0],
scope_name=scope_name)
log_info_str = 'frame info: ' + self.frame_info + '\n'
log_info_str += self.anchors_details()
log_info_str += self.rpn_poposal_details()
self.log_info(self.log_subdir, log_info_str)
# self.log_prediction(batch_top_view, batch_front_view, batch_rgb_images,
# batch_gt_labels, batch_gt_boxes3d,log_rpn=,
# step=self.n_global_step, scope_name=scope_name, print_iou=True,
# loss=(f_cls_loss, f_reg_loss))
self.predict_log(self.log_subdir,
log_rpn=True,
step=self.n_global_step,
scope_name=scope_name,
loss=(f_cls_loss, f_reg_loss),
frame_tag=self.frame_id,
is_train_mode=True)
self.log_msg.write(
'Image log summary use time : {}\n'.format(time.time() - t0))
return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss