def log_prediction(self, batch_top_view, batch_front_view, batch_rgb_images, batch_gt_labels=None, batch_gt_boxes3d=None, print_iou=False, log_rpn=False, step=None, scope_name=''): boxes3d, lables = self.predict(batch_top_view, batch_front_view, batch_rgb_images) self.predict_log(self.log_subdir, log_rpn=log_rpn, step=step, scope_name=scope_name) if type(batch_gt_boxes3d) == np.ndarray and type( batch_gt_labels) == np.ndarray: inds = np.where(batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds], boxes3d) tag = os.path.join(scope_name, 'IOU') self.summary_scalar(value=iou, tag=tag, step=self.n_global_step) except ValueError: iou = -1 print("waring :", sys.exc_info()[0]) if print_iou: self.log_msg.write('\n %s iou: %.5f\n' % (scope_name, iou))
def log_prediction(self, batch_top_view, batch_front_view, batch_rgb_images, batch_gt_labels=None, batch_gt_boxes3d=None, print_iou=False, log_rpn=False, step=None, scope_name='',loss:tuple =None): boxes3d, lables = self.predict(batch_top_view, batch_front_view, batch_rgb_images) self.predict_log(self.log_subdir,log_rpn=log_rpn, step=step, scope_name=scope_name, loss=loss, frame_tag=self.frame_id, is_train_mode=True) if type(batch_gt_boxes3d)==np.ndarray and type(batch_gt_labels)==np.ndarray: inds = np.where(batch_gt_labels[0]!=0) try: iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds], boxes3d) tag = os.path.join('IOU') self.summary_scalar(value=iou, tag=tag, step=self.n_global_step) except ValueError: iou= -1 print("waring :", sys.exc_info()[0]) if print_iou: self.log_msg.write('\n %s iou: %.5f\n' % (self.step_name, iou))
def fit_iteration(self, is_validation=False, summary_it=False, summary_runmeta=False, log_image=False, summary_iou=False, iou_statistic=False): data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = \ self.val_summary_writer if is_validation else self.train_summary_writer self.step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, self.calib = \ data_set.load() # print(self.raw_img.files_path_mapping[self.frame_id]) # resize rgb images self.batch_rgb_images = np.array([cv2.resize(x, (self.rgb_shape[1], self.rgb_shape[0])) \ for x in self.batch_rgb_images]) # fit_iterate log init if log_image: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info() self.log_subdir = self.step_name + '/' + self.time_str self.top_image = data.draw_top_image(self.batch_top_view[0]) # self.top_image = self.top_image_padding(top_image) net = self.net sess = self.sess # put tensorboard inside top_cls_loss = net['top_cls_loss'] top_reg_loss = net['top_reg_loss'] fuse_cls_loss = net['fuse_cls_loss'] fuse_reg_loss = net['fuse_reg_loss'] self.batch_gt_top_boxes = box.box3d_to_top_box( self.batch_gt_boxes3d[0]) ## run propsal generation fd1 = { net['top_view']: self.batch_top_view, net['top_anchors']: self.top_view_anchors, net['top_inside_inds']: self.anchors_inside_inds, blocks.IS_TRAIN_PHASE: True, K.learning_phase(): 1 } self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \ sess.run([net['train_proposals'], net['train_proposal_scores'], net['top_features']], fd1) ## generate train rois for RPN self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \ rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0], self.batch_gt_top_boxes) self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \ rcnn_target(self.batch_proposals, self.batch_gt_labels[0], self.batch_gt_top_boxes, self.batch_gt_boxes3d[0]) # print(self.anchors_details()) # print(self.rpn_poposal_details()) self.batch_rois3d = project_to_roi3d(self.batch_top_rois) self.batch_front_rois = project_to_front_roi(self.batch_rois3d) self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d, self.calib.velo_to_rgb) ## run classification and regression loss ----------- fd2 = { **fd1, net['top_view']: self.batch_top_view, net['front_view']: self.batch_front_view, net['rgb_images']: self.batch_rgb_images, net['top_rois']: self.batch_top_rois, net['front_rois']: self.batch_front_rois, net['rgb_rois']: self.batch_rgb_rois, net['top_inds']: self.batch_top_inds, net['top_pos_inds']: self.batch_top_pos_inds, net['top_labels']: self.batch_top_labels, net['top_targets']: self.batch_top_targets, net['fuse_labels']: self.batch_fuse_labels, net['fuse_targets']: self.batch_fuse_targets, } """ if self.debug_mode: print('\n\nstart debug mode\n\n') debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess) debug_sess.add_tensor_filter('has_inf_or_nan', tf_debug.has_inf_or_nan) t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) """ if summary_it: run_options = None run_metadata = None if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2) self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added validation summary ') else: if summary_runmeta: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata) self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added training summary ') if summary_runmeta: self.train_summary_writer.add_run_metadata( run_metadata, 'step%d' % self.n_global_step) # print('added runtime metadata.') else: if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) else: _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], feed_dict=fd2) if log_image: # t0 = time.time() step_name = 'validation' if is_validation else 'train' scope_name = '%s_iter_%06d' % ( step_name, self.n_global_step - (self.n_global_step % self.iter_debug)) boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, \ self.batch_rgb_images, self.calib.velo_to_rgb) if 0 and set(self.train_target) != set(mv3d_net.top_view_rpn_name): # get iou iou = -1 inds = np.where(self.batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d) if iou_statistic: self.iou_store.append(iou) if summary_iou: iou_aver = sum(self.iou_store) / len(self.iou_store) self.iou_store = [] tag = os.path.join('IOU') self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step) self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver)) except ValueError: # print("waring :", sys.exc_info()[0]) pass #set scope name if iou == -1: scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8))) else: for iou_range in self.log_iou_range: if int(iou * 100) in iou_range: scope_name = os.path.join( scope_name, 'iou_{}'.format(iou_range)) # print('Summary log image, scope name: {}'.format(scope_name)) self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name) log_info_str = 'frame info: ' + self.frame_info + '\n' log_info_str += self.anchors_details() log_info_str += self.rpn_poposal_details() log_info_str += '\n' self.log_info(self.log_subdir, log_info_str) self.predict_log(self.log_subdir, self.calib.velo_to_rgb, log_rpn=True, step=self.n_global_step, scope_name=scope_name, loss=(f_cls_loss, f_reg_loss), frame_tag=self.frame_id, is_train_mode=True) # self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0)) return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def fit_iteration(self, is_validation =False, summary_it=False, summary_runmeta=False, log_image=False, summary_iou=False, iou_statistic=False): data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = \ self.val_summary_writer if is_validation else self.train_summary_writer self.step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load() # fit_iterate log init if log_image: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info() self.log_subdir = self.step_name + '/' + self.time_str self.top_image = data.draw_top_image(self.batch_top_view[0]) # self.top_image = self.top_image_padding(top_image) net = self.net sess = self.sess # put tensorboard inside top_cls_loss = net['top_cls_loss'] top_reg_loss = net['top_reg_loss'] fuse_cls_loss = net['fuse_cls_loss'] fuse_reg_loss = net['fuse_reg_loss'] self.batch_gt_top_boxes = data.box3d_to_top_box(self.batch_gt_boxes3d[0]) ## run propsal generation fd1 = { net['top_view']: self.batch_top_view, net['top_anchors']: self.top_view_anchors, net['top_inside_inds']: self.anchors_inside_inds, blocks.IS_TRAIN_PHASE: True, K.learning_phase(): 1 } self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \ sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1) ## generate train rois for RPN self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \ rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0], self.batch_gt_top_boxes) self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \ fusion_target(self.batch_proposals, self.batch_gt_labels[0], self.batch_gt_top_boxes, self.batch_gt_boxes3d[0]) self.batch_rois3d = project_to_roi3d(self.batch_top_rois) self.batch_front_rois = project_to_front_roi(self.batch_rois3d) self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d) ## run classification and regression loss ----------- fd2 = { **fd1, net['top_view']: self.batch_top_view, net['front_view']: self.batch_front_view, net['rgb_images']: self.batch_rgb_images, net['top_rois']: self.batch_top_rois, net['front_rois']: self.batch_front_rois, net['rgb_rois']: self.batch_rgb_rois, net['top_inds']: self.batch_top_inds, net['top_pos_inds']: self.batch_top_pos_inds, net['top_labels']: self.batch_top_labels, net['top_targets']: self.batch_top_targets, net['fuse_labels']: self.batch_fuse_labels, net['fuse_targets']: self.batch_fuse_targets, } if self.debug_mode: print('\n\nstart debug mode\n\n') debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess) t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) if summary_it: run_options = None run_metadata = None if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2) self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added validation summary ') else: if summary_runmeta: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata) self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added training summary ') if summary_runmeta: self.train_summary_writer.add_run_metadata(run_metadata, 'step%d' % self.n_global_step) # print('added runtime metadata.') else: if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) else: _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], feed_dict=fd2) if log_image: # t0 = time.time() step_name = 'validation' if is_validation else 'train' scope_name = '%s_iter_%06d' % (step_name, self.n_global_step - (self.n_global_step % self.iter_debug)) boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, self.batch_rgb_images) # get iou iou = -1 inds = np.where(self.batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d) if iou_statistic: self.iou_store.append(iou) if summary_iou: iou_aver = sum(self.iou_store)/len(self.iou_store) self.iou_store=[] tag = os.path.join('IOU') self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step) self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver)) except ValueError: # print("waring :", sys.exc_info()[0]) pass #set scope name if iou == -1: scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8))) else: for iou_range in self.log_iou_range: if int(iou*100) in iou_range: scope_name = os.path.join(scope_name , 'iou_{}'.format (iou_range)) # print('Summary log image, scope name: {}'.format(scope_name)) self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name) log_info_str = 'frame info: ' + self.frame_info + '\n' log_info_str += self.anchors_details() log_info_str += self.rpn_poposal_details() self.log_info(self.log_subdir, log_info_str) self.predict_log(self.log_subdir, log_rpn=True, step=self.n_global_step, scope_name=scope_name, loss=(f_cls_loss, f_reg_loss), frame_tag=self.frame_id, is_train_mode=True) # self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0)) return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def fit_iteration(self, batch_rgb_images, batch_top_view, batch_front_view, batch_gt_labels, batch_gt_boxes3d, frame_id, is_validation=False, summary_it=False, summary_runmeta=False, log=False): net = self.net sess = self.sess # put tensorboard inside top_cls_loss = net['top_cls_loss'] top_reg_loss = net['top_reg_loss'] fuse_cls_loss = net['fuse_cls_loss'] fuse_reg_loss = net['fuse_reg_loss'] self.batch_gt_top_boxes = data.box3d_to_top_box(batch_gt_boxes3d[0]) ## run propsal generation fd1 = { net['top_view']: batch_top_view, net['top_anchors']: self.top_view_anchors, net['top_inside_inds']: self.anchors_inside_inds, blocks.IS_TRAIN_PHASE: True, K.learning_phase(): 1 } self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \ sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1) ## generate train rois for RPN self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \ rpn_target(self.top_view_anchors, self.anchors_inside_inds, batch_gt_labels[0], self.batch_gt_top_boxes) self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \ fusion_target(self.batch_proposals, batch_gt_labels[0], self.batch_gt_top_boxes, batch_gt_boxes3d[0]) self.batch_rois3d = project_to_roi3d(self.batch_top_rois) self.batch_front_rois = project_to_front_roi(self.batch_rois3d) self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d) ## run classification and regression loss ----------- fd2 = { **fd1, net['top_view']: batch_top_view, net['front_view']: batch_front_view, net['rgb_images']: batch_rgb_images, net['top_rois']: self.batch_top_rois, net['front_rois']: self.batch_front_rois, net['rgb_rois']: self.batch_rgb_rois, net['top_inds']: self.batch_top_inds, net['top_pos_inds']: self.batch_top_pos_inds, net['top_labels']: self.batch_top_labels, net['top_targets']: self.batch_top_targets, net['fuse_labels']: self.batch_fuse_labels, net['fuse_targets']: self.batch_fuse_targets, } if self.debug_mode: print('\n\nstart debug mode\n\n') debug_sess = tf_debug.LocalCLIDebugWrapperSession(sess) t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) if summary_it: run_options = None run_metadata = None if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2) self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step) print('added validation summary ') else: if summary_runmeta: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata) self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step) print('added training summary ') if summary_runmeta: self.train_summary_writer.add_run_metadata( run_metadata, 'step%d' % self.n_global_step) print('added runtime metadata.') else: if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) else: _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], feed_dict=fd2) if log: t0 = time.time() step_name = 'validation' if is_validation else 'train' scope_name = '%s_iter_%06d' % ( step_name, self.n_global_step - (self.n_global_step % self.iter_debug)) boxes3d, lables = self.predict(batch_top_view, batch_front_view, batch_rgb_images) # get iou iou = -1 if type(batch_gt_boxes3d) == np.ndarray and type( batch_gt_labels) == np.ndarray: inds = np.where(batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(batch_gt_boxes3d[0][inds], boxes3d) tag = os.path.join('IOU') self.summary_scalar(value=iou, tag=tag, step=self.n_global_step) except ValueError: print("waring :", sys.exc_info()[0]) self.log_msg.write('\n %s iou: %.5f\n' % (self.step_name, iou)) #set scope name if iou == -1: scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8))) else: for iou_range in self.log_iou_range: if int(iou * 100) in iou_range: scope_name = os.path.join(scope_name, 'iou_{}'.format(iou_range)) print('Summary log image, scope name: {}'.format(scope_name)) self.log_fusion_net_target(batch_rgb_images[0], scope_name=scope_name) log_info_str = 'frame info: ' + self.frame_info + '\n' log_info_str += self.anchors_details() log_info_str += self.rpn_poposal_details() self.log_info(self.log_subdir, log_info_str) # self.log_prediction(batch_top_view, batch_front_view, batch_rgb_images, # batch_gt_labels, batch_gt_boxes3d,log_rpn=, # step=self.n_global_step, scope_name=scope_name, print_iou=True, # loss=(f_cls_loss, f_reg_loss)) self.predict_log(self.log_subdir, log_rpn=True, step=self.n_global_step, scope_name=scope_name, loss=(f_cls_loss, f_reg_loss), frame_tag=self.frame_id, is_train_mode=True) self.log_msg.write( 'Image log summary use time : {}\n'.format(time.time() - t0)) return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss