def draw_bbox_on_lidar_top(top, boxes3d, one_frame_tag): path = os.path.join(config.cfg.LOG_DIR, 'test', 'top', '%s.png' % one_frame_tag.replace('/', '_')) top_image = data.draw_top_image(top) top_image = data.draw_box3d_on_top(top_image, boxes3d, color=(0, 0, 80)) cv2.imwrite(path, top_image) print('write %s finished' % path)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='',loss:tuple =None, frame_tag='unknown_tag',is_train_mode=False): font = cv2.FONT_HERSHEY_SIMPLEX text_pos = (5, 25) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) # origin top view self.top_image = data.draw_top_image(self.top_view[0]) top_view_log = self.top_image.copy() # add text on origin text = frame_tag cv2.putText(top_view_log, text, text_pos, font, 0.3, (0, 255, 100), 0, cv2.LINE_AA) if log_rpn: rpn_img = self.log_rpn(step=step, scope_name=scope_name, is_train_mode=is_train_mode, tensor_board=False) top_view_log = np.concatenate((top_view_log, rpn_img), 1) # prediction on top predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d) # add fusion loss text text = '' if loss != None: text += 'loss c: %6f r: %6f' % loss cv2.putText(predict_top_view, text, text_pos, font, 0.4, (0, 255, 100), 0, cv2.LINE_AA) # concatenate top_view_log and final prediction top_view_log = np.concatenate((top_view_log, predict_top_view), 1) # new_size = (top_view_log.shape[1] // 2, top_view_log.shape[0] // 2) # top_view_log = cv2.resize(top_view_log, new_size) self.summary_image(top_view_log, scope_name + '/top_view', step=step) # prediction on rgb text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)] prediction_on_rgb = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables) self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name=''): self.top_image = data.draw_top_image(self.top_view[0]) self.top_image = self.top_image_padding(self.top_image) if log_rpn: self.log_rpn(step=step, scope_name=scope_name) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) text_lables = [ 'No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs) ] predict_camera_view = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables) new_size = (predict_camera_view.shape[1] // 2, predict_camera_view.shape[0] // 2) predict_camera_view = cv2.resize(predict_camera_view, new_size) # nud.imsave('predict_camera_view' , predict_camera_view, log_subdir) self.summary_image(predict_camera_view, scope_name + '/predict_camera_view', step=step) predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d) # nud.imsave('predict_top_view' , predict_top_view, log_subdir) self.summary_image(predict_top_view, scope_name + '/predict_top_view', step=step)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='', loss: tuple = None, frame_tag='unknown_tag', is_train_mode=False): font = cv2.FONT_HERSHEY_SIMPLEX text_pos = (5, 25) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) # origin top view self.top_image = data.draw_top_image(self.batch_top_view[0]) top_view_log = self.top_image.copy() # add text on origin text = frame_tag cv2.putText(top_view_log, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA) if log_rpn: rpn_img = self.log_rpn(step=step, scope_name=scope_name, tensor_board=False, draw_rpn_target=False) top_view_log = np.concatenate((top_view_log, rpn_img), 1) # all prediction on top probs, boxes3d = rcnn_nms(self.fuse_probs, self.fuse_deltas, self.rois3d, score_threshold=0.) fusion_proposal_top = data.draw_box3d_on_top(self.top_image, boxes3d) prediction_top = data.draw_box3d_on_top(self.top_image, self.boxes3d) # add fusion loss text text = '' if loss != None: text += 'loss c: %6f r: %6f' % loss cv2.putText(fusion_proposal_top, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA) # concatenate top_view_log and final prediction top_view_log = np.concatenate((top_view_log, fusion_proposal_top), 1) top_view_log = np.concatenate((top_view_log, prediction_top), 1) self.summary_image(top_view_log, scope_name + '/top_view', step=step) # prediction on rgb text_lables = [ 'No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs) ] prediction_on_rgb = nud.draw_box3d_on_camera(self.batch_rgb_images[0], self.boxes3d, text_lables=text_lables) self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
def visualize_result(tag, top_view, rgb, boxes3d, probs, gt_boxes3d=[]): top_image = data.draw_top_image(top_view) top_image = top_image_padding(top_image) text_lables = [ 'No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(probs) ] predict_camera_view = nud.draw_box3d_on_camera(rgb, boxes3d, text_lables=text_lables) predict_top_view = data.draw_box3d_on_top(top_image, boxes3d) # draw gt on camera and top view: if len(gt_boxes3d) > 0: # array size > 1 cannot directly used in if predict_top_view = data.draw_box3d_on_top(predict_top_view, gt_boxes3d, color=(0, 0, 255)) predict_camera_view = draw_box3d_on_camera(predict_camera_view, gt_boxes3d, color=(0, 0, 255)) new_size = (predict_camera_view.shape[1] // 2, predict_camera_view.shape[0] // 2) predict_camera_view = cv2.resize(predict_camera_view, new_size) cv2.imwrite(os.path.join(args.target_dir, 'image', tag + 'rgb_.png'), predict_camera_view) cv2.imwrite(os.path.join(args.target_dir, 'image', tag + 'top_.png'), predict_top_view)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='', loss: tuple = None, frame_tag='unknown_tag', is_train_mode=False): font = cv2.FONT_HERSHEY_SIMPLEX text_pos = (5, 25) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) # origin top view self.top_image = data.draw_top_image(self.top_view[0]) top_view_log = self.top_image.copy() # add text on origin text = frame_tag cv2.putText(top_view_log, text, text_pos, font, 0.3, (0, 255, 100), 0, cv2.LINE_AA) if log_rpn: rpn_img = self.log_rpn(step=step, scope_name=scope_name, is_train_mode=is_train_mode, tensor_board=False) top_view_log = np.concatenate((top_view_log, rpn_img), 1) # prediction on top predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d) # add fusion loss text text = '' if loss != None: text += 'loss c: %6f r: %6f' % loss cv2.putText(predict_top_view, text, text_pos, font, 0.4, (0, 255, 100), 0, cv2.LINE_AA) # concatenate top_view_log and final prediction top_view_log = np.concatenate((top_view_log, predict_top_view), 1) # new_size = (top_view_log.shape[1] // 2, top_view_log.shape[0] // 2) # top_view_log = cv2.resize(top_view_log, new_size) self.summary_image(top_view_log, scope_name + '/top_view', step=step) # prediction on rgb text_lables = [ 'No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs) ] prediction_on_rgb = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables) self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name=''): self.top_image = data.draw_top_image(self.top_view[0]) self.top_image = self.top_image_padding(self.top_image) if log_rpn: self.log_rpn(step=step ,scope_name=scope_name) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)] predict_camera_view = nud.draw_box3d_on_camera(self.rgb_image[0], self.boxes3d, text_lables=text_lables) new_size = (predict_camera_view.shape[1] // 2, predict_camera_view.shape[0] // 2) predict_camera_view = cv2.resize(predict_camera_view, new_size) # nud.imsave('predict_camera_view' , predict_camera_view, log_subdir) self.summary_image(predict_camera_view, scope_name + '/predict_camera_view', step=step) predict_top_view = data.draw_box3d_on_top(self.top_image, self.boxes3d) # nud.imsave('predict_top_view' , predict_top_view, log_subdir) self.summary_image(predict_top_view, scope_name + '/predict_top_view', step=step)
def log_image(self, step, frame_tag, prefix, summary_writer): font = cv2.FONT_HERSHEY_SIMPLEX self.top_image = data.draw_top_image(self.batch_top_view[0]) top_view_log = self.top_image.copy() # add text on origin text = frame_tag for i, line in enumerate(text.split("/")): y = 25 + i * 20 cv2.putText(top_view_log, line, (5, y), font, 0.5, (0, 255, 100), 0, cv2.LINE_AA) # draw groundtruth on topview for gt_bbox in self.batch_gt_boxes3d[:]: topview = net.processing.boxes3d.draw_box3d_on_top( top_view_log, gt_bbox) self.summary_image(topview, prefix + '/top_view', step=step, summary_writer=summary_writer)
def predict_log(self, log_subdir, log_rpn=False, step=None, scope_name='',loss:tuple =None, frame_tag='unknown_tag',is_train_mode=False): font = cv2.FONT_HERSHEY_SIMPLEX text_pos = (5, 25) self.log_fusion_net_detail(log_subdir, self.fuse_probs, self.fuse_deltas) # origin top view self.top_image = data.draw_top_image(self.batch_top_view[0]) top_view_log = self.top_image.copy() # add text on origin text = frame_tag cv2.putText(top_view_log, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA) if log_rpn: rpn_img = self.log_rpn(step=step, scope_name=scope_name, tensor_board=False,draw_rpn_target=False) top_view_log = np.concatenate((top_view_log, rpn_img), 1) # all prediction on top probs, boxes3d = rcnn_nms(self.fuse_probs, self.fuse_deltas,self.rois3d, score_threshold=0.) fusion_proposal_top = data.draw_box3d_on_top(self.top_image, boxes3d,scores=probs,thickness=0) prediction_top = data.draw_box3d_on_top(self.top_image, self.boxes3d, scores=self.probs, thickness=0) # add fusion loss text text = '' if loss != None: text += 'loss c: %6f r: %6f' % loss cv2.putText(fusion_proposal_top, text, text_pos, font, 0.5, (0, 255, 100), 0, cv2.LINE_AA) # concatenate top_view_log and final prediction top_view_log = np.concatenate((top_view_log, fusion_proposal_top), 1) top_view_log = np.concatenate((top_view_log, prediction_top), 1) self.summary_image(top_view_log, scope_name + '/top_view', step=step) # prediction on rgb text_lables = ['No.%d class:1 prob: %.4f' % (i, prob) for i, prob in enumerate(self.probs)] prediction_on_rgb = nud.draw_box3d_on_camera(self.batch_rgb_images[0], self.boxes3d, text_lables=text_lables) self.summary_image(prediction_on_rgb, scope_name + '/prediction_on_rgb', step=step)
# load objs = raw_tracklet.load(one_frame_tag) rgb = raw_img.load(one_frame_tag) lidar = raw_lidar.load(one_frame_tag) # preprocess rgb = preprocess.rgb(rgb) top = preprocess.lidar_to_top(lidar) boxes3d = [preprocess.bbox3d(obj) for obj in objs] labels = [preprocess.label(obj) for obj in objs] # dump rgb img = draw.draw_box3d_on_camera(rgb, boxes3d) new_size = (img.shape[1] // 3, img.shape[0] // 3) img = cv2.resize(img, new_size) path = os.path.join(config.cfg.LOG_DIR, 'test', 'rgb', '%s.png' % one_frame_tag.replace('/', '_')) cv2.imwrite(path, img) print('write %s finished' % path) # draw bbox on top image path = os.path.join(config.cfg.LOG_DIR, 'test', 'top', '%s.png' % one_frame_tag.replace('/', '_')) top_image = data.draw_top_image(top) top_image = data.draw_box3d_on_top(top_image, boxes3d, color=(0, 0, 80)) cv2.imwrite(path, top_image) print('write %s finished' % path)
def __call__(self, max_iter=1000, train_set =None, validation_set =None): sess = self.sess net = self.net with sess.as_default(): #for init model batch_size=1 validation_step=40 ckpt_save_step=200 if cfg.TRAINING_TIMER: time_it = timer() # start training here ######################################################################################### self.log_msg.write('iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n') self.log_msg.write('-------------------------------------------------------------------------------------\n') for iter in range(max_iter): is_validation = False summary_it = False summary_runmeta = False print_loss = False log_this_iter = False # set fit flag if iter % validation_step == 0: summary_it,is_validation,print_loss = True,True,True # summary validation loss if (iter+1) % validation_step == 0: summary_it,print_loss = True,True # summary train loss if iter % 5 == 0: print_loss = True #print train loss if 1 and iter%300 == 3: summary_it,summary_runmeta = True,True if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0: log_this_iter = True print('summary log image') if iter % self.iter_debug == 0: is_validation =False else: is_validation =True data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load(batch_size, shuffled=True) # # if self.batch_data_is_invalid(self.batch_gt_boxes3d[0]): # continue # for keeping all gt labels and gt boxes inside range, and discard gt out of selected range. is_gt_inside_range, batch_gt_labels_in_range, batch_gt_boxes3d_in_range = \ self.keep_gt_inside_range(self.batch_gt_labels[0], self.batch_gt_boxes3d[0]) if not is_gt_inside_range: continue # todo current support only batch_size == 1 self.batch_gt_labels = np.zeros((1, batch_gt_labels_in_range.shape[0]), dtype=np.int32) self.batch_gt_boxes3d = np.zeros((1, batch_gt_labels_in_range.shape[0], 8, 3), dtype=np.float32) self.batch_gt_labels[0] = batch_gt_labels_in_range self.batch_gt_boxes3d[0] = batch_gt_boxes3d_in_range # fit_iterate log init if log_this_iter: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info(self.frame_id)[0] self.log_subdir = step_name + '/' + self.time_str top_image = data.draw_top_image(self.batch_top_view[0]) self.top_image = self.top_image_padding(top_image) # fit t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \ self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view, self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, is_validation =is_validation, summary_it=summary_it, summary_runmeta=summary_runmeta, log=log_this_iter) if print_loss: self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \ (step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss)) if iter%ckpt_save_step==0: # saver.save(sess, pretrained_model_path) print('save_weights') self.save_weights(self.train_target) if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \ (time_it.time_diff_per_n_loops(), ckpt_save_step)) self.gc() self.n_global_step += 1 if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \ (time_it.total_time())) self.save_progress()
def __call__(self, max_iter=1000, train_set=None, validation_set=None): sess = self.sess net = self.net with sess.as_default(): #for init model batch_size = 1 validation_step = 40 ckpt_save_step = 200 if cfg.TRAINING_TIMER: time_it = timer() # start training here ######################################################################################### self.log_msg.write( 'iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n' ) self.log_msg.write( '-------------------------------------------------------------------------------------\n' ) for iter in range(max_iter): is_validation = False summary_it = False summary_runmeta = False print_loss = False log_this_iter = False # set fit flag if iter % validation_step == 0: summary_it, is_validation, print_loss = True, True, True # summary validation loss if (iter + 1) % validation_step == 0: summary_it, print_loss = True, True # summary train loss if iter % 5 == 0: print_loss = True #print train loss if 1 and iter % 300 == 3: summary_it, summary_runmeta = True, True if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0: log_this_iter = True print('summary log image') if iter % self.iter_debug == 0: is_validation = False else: is_validation = True data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load(batch_size, shuffled=True) # # if self.batch_data_is_invalid(self.batch_gt_boxes3d[0]): # continue # for keeping all gt labels and gt boxes inside range, and discard gt out of selected range. is_gt_inside_range, batch_gt_labels_in_range, batch_gt_boxes3d_in_range = \ self.keep_gt_inside_range(self.batch_gt_labels[0], self.batch_gt_boxes3d[0]) if not is_gt_inside_range: continue # todo current support only batch_size == 1 self.batch_gt_labels = np.zeros( (1, batch_gt_labels_in_range.shape[0]), dtype=np.int32) self.batch_gt_boxes3d = np.zeros( (1, batch_gt_labels_in_range.shape[0], 8, 3), dtype=np.float32) self.batch_gt_labels[0] = batch_gt_labels_in_range self.batch_gt_boxes3d[0] = batch_gt_boxes3d_in_range # fit_iterate log init if log_this_iter: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info(self.frame_id)[0] self.log_subdir = step_name + '/' + self.time_str top_image = data.draw_top_image(self.batch_top_view[0]) self.top_image = self.top_image_padding(top_image) # fit t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \ self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view, self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, is_validation =is_validation, summary_it=summary_it, summary_runmeta=summary_runmeta, log=log_this_iter) if print_loss: self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \ (step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss)) if iter % ckpt_save_step == 0: # saver.save(sess, pretrained_model_path) print('save_weights') self.save_weights(self.train_target) if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \ (time_it.time_diff_per_n_loops(), ckpt_save_step)) self.gc() self.n_global_step += 1 if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \ (time_it.total_time())) self.save_progress()
def fit_iteration(self, is_validation=False, summary_it=False, summary_runmeta=False, log_image=False, summary_iou=False, iou_statistic=False): data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = \ self.val_summary_writer if is_validation else self.train_summary_writer self.step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, self.calib = \ data_set.load() # print(self.raw_img.files_path_mapping[self.frame_id]) # resize rgb images self.batch_rgb_images = np.array([cv2.resize(x, (self.rgb_shape[1], self.rgb_shape[0])) \ for x in self.batch_rgb_images]) # fit_iterate log init if log_image: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info() self.log_subdir = self.step_name + '/' + self.time_str self.top_image = data.draw_top_image(self.batch_top_view[0]) # self.top_image = self.top_image_padding(top_image) net = self.net sess = self.sess # put tensorboard inside top_cls_loss = net['top_cls_loss'] top_reg_loss = net['top_reg_loss'] fuse_cls_loss = net['fuse_cls_loss'] fuse_reg_loss = net['fuse_reg_loss'] self.batch_gt_top_boxes = box.box3d_to_top_box( self.batch_gt_boxes3d[0]) ## run propsal generation fd1 = { net['top_view']: self.batch_top_view, net['top_anchors']: self.top_view_anchors, net['top_inside_inds']: self.anchors_inside_inds, blocks.IS_TRAIN_PHASE: True, K.learning_phase(): 1 } self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \ sess.run([net['train_proposals'], net['train_proposal_scores'], net['top_features']], fd1) ## generate train rois for RPN self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \ rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0], self.batch_gt_top_boxes) self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \ rcnn_target(self.batch_proposals, self.batch_gt_labels[0], self.batch_gt_top_boxes, self.batch_gt_boxes3d[0]) # print(self.anchors_details()) # print(self.rpn_poposal_details()) self.batch_rois3d = project_to_roi3d(self.batch_top_rois) self.batch_front_rois = project_to_front_roi(self.batch_rois3d) self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d, self.calib.velo_to_rgb) ## run classification and regression loss ----------- fd2 = { **fd1, net['top_view']: self.batch_top_view, net['front_view']: self.batch_front_view, net['rgb_images']: self.batch_rgb_images, net['top_rois']: self.batch_top_rois, net['front_rois']: self.batch_front_rois, net['rgb_rois']: self.batch_rgb_rois, net['top_inds']: self.batch_top_inds, net['top_pos_inds']: self.batch_top_pos_inds, net['top_labels']: self.batch_top_labels, net['top_targets']: self.batch_top_targets, net['fuse_labels']: self.batch_fuse_labels, net['fuse_targets']: self.batch_fuse_targets, } """ if self.debug_mode: print('\n\nstart debug mode\n\n') debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess) debug_sess.add_tensor_filter('has_inf_or_nan', tf_debug.has_inf_or_nan) t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) """ if summary_it: run_options = None run_metadata = None if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2) self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added validation summary ') else: if summary_runmeta: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata) self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added training summary ') if summary_runmeta: self.train_summary_writer.add_run_metadata( run_metadata, 'step%d' % self.n_global_step) # print('added runtime metadata.') else: if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) else: _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], feed_dict=fd2) if log_image: # t0 = time.time() step_name = 'validation' if is_validation else 'train' scope_name = '%s_iter_%06d' % ( step_name, self.n_global_step - (self.n_global_step % self.iter_debug)) boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, \ self.batch_rgb_images, self.calib.velo_to_rgb) if 0 and set(self.train_target) != set(mv3d_net.top_view_rpn_name): # get iou iou = -1 inds = np.where(self.batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d) if iou_statistic: self.iou_store.append(iou) if summary_iou: iou_aver = sum(self.iou_store) / len(self.iou_store) self.iou_store = [] tag = os.path.join('IOU') self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step) self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver)) except ValueError: # print("waring :", sys.exc_info()[0]) pass #set scope name if iou == -1: scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8))) else: for iou_range in self.log_iou_range: if int(iou * 100) in iou_range: scope_name = os.path.join( scope_name, 'iou_{}'.format(iou_range)) # print('Summary log image, scope name: {}'.format(scope_name)) self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name) log_info_str = 'frame info: ' + self.frame_info + '\n' log_info_str += self.anchors_details() log_info_str += self.rpn_poposal_details() log_info_str += '\n' self.log_info(self.log_subdir, log_info_str) self.predict_log(self.log_subdir, self.calib.velo_to_rgb, log_rpn=True, step=self.n_global_step, scope_name=scope_name, loss=(f_cls_loss, f_reg_loss), frame_tag=self.frame_id, is_train_mode=True) # self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0)) return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def pred_and_save(tracklet_pred_dir, dataset, generate_video=False, frame_offset=16, log_tag=None, weights_tag=None): # Tracklet_saver will check whether the file already exists. tracklet = Tracklet_saver(tracklet_pred_dir) os.makedirs (os.path.join(log_dir,'image'),exist_ok=True) top_shape, front_shape, rgb_shape=dataset.get_shape() predict=mv3d.Predictor(top_shape, front_shape, rgb_shape, log_tag=log_tag, weights_tag=weights_tag) if generate_video: vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir,'output.mp4')) # timer timer_step = 100 if cfg.TRACKING_TIMER: time_it = timer() frame_num = 0 for i in range(dataset.size): rgb, top, front, _, _,_= dataset.load(1) frame_num = i - frame_offset if frame_num < 0: continue boxes3d,probs=predict(top, front, rgb) predict.dump_log(log_subdir=log_subdir,n_frame=i) # time timer_step iterations. Turn it on/off in config.py if cfg.TRACKING_TIMER and i%timer_step ==0 and i!=0: predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \ (time_it.time_diff_per_n_loops(), timer_step)) # for debugging: save image and show image. top_image = draw_top_image(top[0]) rgb_image = rgb[0] if len(boxes3d)!=0: top_image = draw_box3d_on_top(top_image, boxes3d[:,:,:], color=(80, 80, 0), thickness=3) rgb_image = draw_box3d_on_camera(rgb_image, boxes3d[:, :, :], color=(0, 0, 80), thickness=3) translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :]) #todo: remove it after gtbox is ok size[:,1:3] = size[:,1:3]/cfg.TRACKLET_GTBOX_LENGTH_SCALE for j in range(len(translation)): if 0<translation[j,1]<8: tracklet.add_tracklet(frame_num, size[j], translation[j], rotation[j]) resize_scale=top_image.shape[0]/rgb_image.shape[0] rgb_image = cv2.resize(rgb_image,(int(rgb_image.shape[1]*resize_scale), top_image.shape[0])) rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB) new_image = np.concatenate((top_image, rgb_image), axis = 1) cv2.imwrite(os.path.join(log_dir,'image','%5d_image.jpg'%i), new_image) if generate_video: vid_in.writeFrame(new_image) vid_in.close() tracklet.write_tracklet() predict.dump_weigths(os.path.join(log_dir, 'pretrained_model')) if cfg.TRACKING_TIMER: predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \ (time_it.total_time())) print("tracklet file named tracklet_labels.xml is written successfully.") return tracklet.path
def iteration_summary(self, prefix, summaryWriter): """ Method for creating numerous image summaries. :param prefix: Can be 'training' or 'validation' :param summaryWriter: Writer instance either for training or validation """ # get the indices of pos+neg, positive anchors as well as labels and regression targets top_inds, pos_inds, labels, targets = self.network.get_targets( self.batch_gt_labels, self.batch_gt_boxes3d) top = data.draw_top_image(self.batch_top_view[0]).copy() self.log_image(step=self.n_global_step, prefix=prefix, frame_tag=self.frame_id, summary_writer=summaryWriter) proposals, scores, probs = self.predict(self.batch_top_view, self.batch_rgb_images) nms_proposals, nms_scores = self.post_process(proposals, probs, minProb=0.1) ###################### ## probabilities ## ###################### for i in range(1, cfg.NUM_CLASSES): # reshape from [-1, NUM_CLASSES] # @TODO don't make the reshape fixed self.summary_image(np.reshape(probs[::2, i], (176 * 2, 200 * 2)), prefix + '/probabilities' + str(i) + '_Up', summaryWriter, step=self.n_global_step) self.summary_image(np.reshape(probs[1::2, i], (176 * 2, 200 * 2)), prefix + '/probabilities' + str(i) + '_Right', summaryWriter, step=self.n_global_step) ###################### ## proposals ## ###################### proposalIdx = np.where(np.argmax(probs, axis=1) != 0) numProposals = np.sum(proposalIdx) Boxes3d = [] if (numProposals > 0): Boxes3d = net.processing.boxes.box_transform_voxelnet_inv( proposals[proposalIdx], self.network.anchors[proposalIdx]) topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d) self.summary_image(topbox, prefix + '/proposalBoxes', summaryWriter, step=self.n_global_step) ###################### ## top 10 proposals ## ###################### sortedIdx = np.argsort(-probs[:, 1].flatten()) top10Proposals = proposals[sortedIdx[:10]] ### top visualization ### Boxes3d = net.processing.boxes.box_transform_voxelnet_inv( top10Proposals, self.network.anchors[sortedIdx[:10]]) topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d) self.summary_image(topbox, prefix + '/top10proposalBoxes', summaryWriter, step=self.n_global_step) ####################### ## nms visualization ## ####################### topbox = top for i in range(0, cfg.NUM_CLASSES - 1): topbox = net.processing.boxes3d.draw_box3d_on_top( topbox, nms_proposals[i][:10, :, :], scores=nms_scores[i]) self.summary_image(topbox, prefix + '/top10NMSBoxes', summaryWriter, step=self.n_global_step) ### to rgb image ### colors = [(255, 0, 255), (255, 0, 0), (0, 255, 0), (0, 0, 255)] rgb_image = self.batch_rgb_images[0] for i in range(0, cfg.NUM_CLASSES - 1): text_labels = ['Prob: %.4f' % (prob) for prob in nms_scores[i]] rgb_image = nud.draw_box3d_on_camera(rgb_image, nms_proposals[i][:, :, :], text_lables=text_labels, color=colors[i]) # add the groundtruth # Boxes3d = net.processing.boxes.box_transform_voxelnet_inv(targets, self.voxelnet.anchors[pos_inds]) # rgb_image = nud.draw_box3d_on_camera(rgb_image, Boxes3d, # text_lables=text_labels, color=(0, 255, 255)) self.summary_image(rgb_image, prefix + '/prediction_on_rgb', summaryWriter, step=self.n_global_step) ###################### ## groundtruth ## ###################### Boxes3d = net.processing.boxes.box_transform_voxelnet_inv( targets, self.network.anchors[pos_inds]) topbox = net.processing.boxes3d.draw_box3d_on_top(top, Boxes3d) self.summary_image(topbox, prefix + '/Targets', summaryWriter, step=self.n_global_step)
def __call__(self, max_iter=1000, train_set=None, validation_set=None): sess = self.sess net = self.net with sess.as_default(): #for init model batch_size = 1 validation_step = 40 ckpt_save_step = 200 if cfg.TRAINING_TIMER: time_it = timer() # start training here ######################################################################################### self.log_msg.write( 'iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n' ) self.log_msg.write( '-------------------------------------------------------------------------------------\n' ) for iter in range(max_iter): is_validation = False summary_it = False summary_runmeta = False print_loss = False log_this_iter = False # set fit flag if iter % validation_step == 0: summary_it, is_validation, print_loss = True, True, True # summary validation loss if (iter + 1) % validation_step == 0: summary_it, print_loss = True, True # summary train loss if iter % 20 == 0: print_loss = True #print train loss if 1 and iter % 300 == 0: summary_it, summary_runmeta = True, True if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0: log_this_iter = True print('Summary log image') if iter % self.iter_debug == 0: is_validation = False else: is_validation = True data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load() # fit_iterate log init if log_this_iter: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info() self.log_subdir = step_name + '/' + self.time_str top_image = data.draw_top_image(self.batch_top_view[0]) self.top_image = self.top_image_padding(top_image) # fit t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \ self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view, self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, is_validation =is_validation, summary_it=summary_it, summary_runmeta=summary_runmeta, log=log_this_iter) if print_loss: self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \ (step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss)) if iter % ckpt_save_step == 0: self.save_weights(self.train_target) if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \ (time_it.time_diff_per_n_loops(), ckpt_save_step)) self.gc() self.n_global_step += 1 if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \ (time_it.total_time())) self.save_progress()
def fit_iteration(self, is_validation =False, summary_it=False, summary_runmeta=False, log_image=False, summary_iou=False, iou_statistic=False): data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = \ self.val_summary_writer if is_validation else self.train_summary_writer self.step_name = 'validation' if is_validation else 'training' # load dataset self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load() # fit_iterate log init if log_image: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info() self.log_subdir = self.step_name + '/' + self.time_str self.top_image = data.draw_top_image(self.batch_top_view[0]) # self.top_image = self.top_image_padding(top_image) net = self.net sess = self.sess # put tensorboard inside top_cls_loss = net['top_cls_loss'] top_reg_loss = net['top_reg_loss'] fuse_cls_loss = net['fuse_cls_loss'] fuse_reg_loss = net['fuse_reg_loss'] self.batch_gt_top_boxes = data.box3d_to_top_box(self.batch_gt_boxes3d[0]) ## run propsal generation fd1 = { net['top_view']: self.batch_top_view, net['top_anchors']: self.top_view_anchors, net['top_inside_inds']: self.anchors_inside_inds, blocks.IS_TRAIN_PHASE: True, K.learning_phase(): 1 } self.batch_proposals, self.batch_proposal_scores, self.batch_top_features = \ sess.run([net['proposals'], net['proposal_scores'], net['top_features']], fd1) ## generate train rois for RPN self.batch_top_inds, self.batch_top_pos_inds, self.batch_top_labels, self.batch_top_targets = \ rpn_target(self.top_view_anchors, self.anchors_inside_inds, self.batch_gt_labels[0], self.batch_gt_top_boxes) self.batch_top_rois, self.batch_fuse_labels, self.batch_fuse_targets = \ fusion_target(self.batch_proposals, self.batch_gt_labels[0], self.batch_gt_top_boxes, self.batch_gt_boxes3d[0]) self.batch_rois3d = project_to_roi3d(self.batch_top_rois) self.batch_front_rois = project_to_front_roi(self.batch_rois3d) self.batch_rgb_rois = project_to_rgb_roi(self.batch_rois3d) ## run classification and regression loss ----------- fd2 = { **fd1, net['top_view']: self.batch_top_view, net['front_view']: self.batch_front_view, net['rgb_images']: self.batch_rgb_images, net['top_rois']: self.batch_top_rois, net['front_rois']: self.batch_front_rois, net['rgb_rois']: self.batch_rgb_rois, net['top_inds']: self.batch_top_inds, net['top_pos_inds']: self.batch_top_pos_inds, net['top_labels']: self.batch_top_labels, net['top_targets']: self.batch_top_targets, net['fuse_labels']: self.batch_fuse_labels, net['fuse_targets']: self.batch_fuse_targets, } if self.debug_mode: print('\n\nstart debug mode\n\n') debug_sess=tf_debug.LocalCLIDebugWrapperSession(sess) t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ debug_sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) if summary_it: run_options = None run_metadata = None if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], fd2) self.val_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added validation summary ') else: if summary_runmeta: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss, tb_sum_val = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss, self.summ], feed_dict=fd2, options=run_options, run_metadata=run_metadata) self.train_summary_writer.add_summary(tb_sum_val, self.n_global_step) # print('added training summary ') if summary_runmeta: self.train_summary_writer.add_run_metadata(run_metadata, 'step%d' % self.n_global_step) # print('added runtime metadata.') else: if is_validation: t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], fd2) else: _, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss = \ sess.run([self.solver_step, top_cls_loss, top_reg_loss, fuse_cls_loss, fuse_reg_loss], feed_dict=fd2) if log_image: # t0 = time.time() step_name = 'validation' if is_validation else 'train' scope_name = '%s_iter_%06d' % (step_name, self.n_global_step - (self.n_global_step % self.iter_debug)) boxes3d, lables = self.predict(self.batch_top_view, self.batch_front_view, self.batch_rgb_images) # get iou iou = -1 inds = np.where(self.batch_gt_labels[0] != 0) try: iou = box.boxes3d_score_iou(self.batch_gt_boxes3d[0][inds], boxes3d) if iou_statistic: self.iou_store.append(iou) if summary_iou: iou_aver = sum(self.iou_store)/len(self.iou_store) self.iou_store=[] tag = os.path.join('IOU') self.summary_scalar(value=iou_aver, tag=tag, step=self.n_global_step) self.log_msg.write('\n %s iou average: %.5f\n' % (self.step_name, iou_aver)) except ValueError: # print("waring :", sys.exc_info()[0]) pass #set scope name if iou == -1: scope_name = os.path.join(scope_name, 'iou_error'.format(range(5, 8))) else: for iou_range in self.log_iou_range: if int(iou*100) in iou_range: scope_name = os.path.join(scope_name , 'iou_{}'.format (iou_range)) # print('Summary log image, scope name: {}'.format(scope_name)) self.log_fusion_net_target(self.batch_rgb_images[0], scope_name=scope_name) log_info_str = 'frame info: ' + self.frame_info + '\n' log_info_str += self.anchors_details() log_info_str += self.rpn_poposal_details() self.log_info(self.log_subdir, log_info_str) self.predict_log(self.log_subdir, log_rpn=True, step=self.n_global_step, scope_name=scope_name, loss=(f_cls_loss, f_reg_loss), frame_tag=self.frame_id, is_train_mode=True) # self.log_msg.write('Image log summary use time : {}\n'.format(time.time() - t0)) return t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss
def pred_and_save(tracklet_pred_dir, dataset, generate_video=False, frame_offset=16, log_tag=None, weights_tag=None): # Tracklet_saver will check whether the file already exists. tracklet = Tracklet_saver(tracklet_pred_dir, 'pred') os.makedirs(os.path.join(log_dir, 'image'), exist_ok=True) gt_tracklet = Tracklet_saver(tracklet_pred_dir, 'gt') top_shape, front_shape, rgb_shape = dataset.get_shape() predict = mv3d.Predictor(top_shape, front_shape, rgb_shape, log_tag=log_tag, weights_tag=weights_tag) if generate_video: vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir, 'output.mp4')) # timer timer_step = 100 if cfg.TRACKING_TIMER: time_it = timer() print('dataset.size') print(dataset.size) lenght = [] gt_lenght = [] frame_num = 0 for i in range(dataset.size if fast_test == False else frame_offset + 1): rgb, top, front, _, _, _ = dataset.load(size=1) frame_num = i - frame_offset print('frame_num') print(frame_num) if frame_num < 0: continue gt_boxes3d_tmp = np.load( '/home/mohsen/Desktop/MV3D/data/preprocessed/kitti/gt_boxes3d/object3d/test/%05d.npy' % i) #remove gt boxes with hiegh less than 40 gt_boxes3d_list = [] for gt_box3d_tmp in gt_boxes3d_tmp: # if gt_box3d_tmp[0,0]>0: gt_box3d_tmp_list = [] gt_box3d_tmp_list.append(gt_box3d_tmp) gt_project = box3d.box3d_to_rgb_box(gt_box3d_tmp_list) if abs(gt_project[0][0, 1] - gt_project[0][4, 1]) >= 40: gt_box3d = gt_box3d_tmp gt_boxes3d_list.append(gt_box3d) gt_boxes3d = np.array(gt_boxes3d_list) # gt_boxes3d = gt_boxes3d_tmp ##################################### boxes3d_tmp, probs = predict(top, front, rgb) predict.dump_log(log_subdir=log_subdir, n_frame=i) # time timer_step iterations. Turn it on/off in config.py if cfg.TRACKING_TIMER and i % timer_step == 0 and i != 0: predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \ (time_it.time_diff_per_n_loops(), timer_step)) # for debugging: save image and show image. top_image = draw_top_image(top[0]) rgb_image = rgb[0] if len(gt_boxes3d) != 0: gt_lenght.append(len(gt_boxes3d)) gt_translation, gt_size, gt_rotation = boxes3d_decompose( gt_boxes3d[:, :, :]) # todo: remove it after gtbox is ok gt_size[:, 1:3] = gt_size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE for j in range(len(gt_translation)): gt_tracklet.add_tracklet(frame_num, gt_size[j], gt_translation[j], gt_rotation[j]) #remove predicted boxes with hiegh less than 40 boxes3d_list = [] for box3d_tmp in boxes3d_tmp: # if box3d_tmp[0, 0] > 0: box3d_tmp_list = [] box3d_tmp_list.append(box3d_tmp) project = box3d.box3d_to_rgb_box(box3d_tmp_list) if abs(project[0][0, 1] - project[0][4, 1]) >= 40: print(project[0][0, 1] - project[0][4, 1]) pred_box3d = box3d_tmp boxes3d_list.append(pred_box3d) boxes3d = np.array(boxes3d_list) # boxes3d = boxes3d_tmp ##################################### print('sizes') print(np.size(boxes3d)) print(gt_boxes3d) print(np.size(gt_boxes3d)) if len(boxes3d) != 0: lenght.append(len(boxes3d)) top_image = draw_box3d_on_top(top_image, boxes3d[:, :, :], color=(80, 80, 0), thickness=3) rgb_image = draw_box3d_on_camera(rgb_image, boxes3d[:, :, :], color=(0, 0, 80), thickness=3) if len(gt_boxes3d) != 0: rgb_image = draw_box3d_on_camera(rgb_image, gt_boxes3d[:, :, :], color=(0, 80, 0), thickness=3) translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :]) # todo: remove it after gtbox is ok size[:, 1:3] = size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE for j in range(len(translation)): tracklet.add_tracklet(frame_num, size[j], translation[j], rotation[j]) resize_scale = top_image.shape[0] / rgb_image.shape[0] rgb_image = cv2.resize( rgb_image, (int(rgb_image.shape[1] * resize_scale), top_image.shape[0])) rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB) new_image = np.concatenate((top_image, rgb_image), axis=1) cv2.imwrite(os.path.join(log_dir, 'image', '%5d_image.jpg' % i), new_image) if generate_video: vid_in.writeFrame(new_image) vid_in.close() print(lenght) print(sum(lenght)) tracklet.write_tracklet() predict.dump_weigths(os.path.join(log_dir, 'pretrained_model')) print(gt_lenght) print(sum(gt_lenght)) gt_tracklet.write_tracklet() if cfg.TRACKING_TIMER: predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \ (time_it.total_time())) print("tracklet file named tracklet_labels.xml is written successfully.") return tracklet.path, gt_tracklet.path
# num libs import math import random import numpy as np import cv2 import config import data import net.utility.draw as draw if __name__ == '__main__': # preprocessed_dir=config.cfg.PREPROCESSED_DATA_SETS_DIR preprocessed_dir=config.cfg.PREPROCESSING_DATA_SETS_DIR dataset='/1/15/00070.npy' top_view_dir =preprocessed_dir+ '/top'+dataset gt_boxes3d_dir =preprocessed_dir+'/gt_boxes3d'+dataset top=np.load(top_view_dir) gt_boxes3d=np.load(gt_boxes3d_dir) top_img=data.draw_top_image(top) top_img_marked=data.draw_box3d_on_top(top_img,gt_boxes3d) draw.imsave('top_img_marked',top_img_marked,'debug') print('top_img_marked dump finished!!')
def pred_and_save(tracklet_pred_dir, dataset, generate_video=False, frame_offset=16, log_tag=None, weights_tag=None): # Tracklet_saver will check whether the file already exists. tracklet = Tracklet_saver(tracklet_pred_dir) os.makedirs(os.path.join(log_dir, 'image'), exist_ok=True) top_shape, front_shape, rgb_shape = dataset.get_shape() predict = mv3d.Predictor(top_shape, front_shape, rgb_shape, log_tag=log_tag, weights_tag=weights_tag) if generate_video: vid_in = skvideo.io.FFmpegWriter(os.path.join(log_dir, 'output.mp4')) # timer timer_step = 100 if cfg.TRACKING_TIMER: time_it = timer() frame_num = 0 for i in range(dataset.size if fast_test == False else frame_offset + 1): rgb, top, front, _, _, _ = dataset.load() frame_num = i - frame_offset if frame_num < 0: continue boxes3d, probs = predict(top, front, rgb) predict.dump_log(log_subdir=log_subdir, n_frame=i) # time timer_step iterations. Turn it on/off in config.py if cfg.TRACKING_TIMER and i % timer_step == 0 and i != 0: predict.track_log.write('It takes %0.2f secs for inferring %d frames. \n' % \ (time_it.time_diff_per_n_loops(), timer_step)) # for debugging: save image and show image. top_image = draw_top_image(top[0]) rgb_image = rgb[0] if len(boxes3d) != 0: top_image = draw_box3d_on_top(top_image, boxes3d[:, :, :], color=(80, 80, 0), thickness=3) rgb_image = draw_box3d_on_camera(rgb_image, boxes3d[:, :, :], color=(0, 0, 80), thickness=3) translation, size, rotation = boxes3d_decompose(boxes3d[:, :, :]) # todo: remove it after gtbox is ok size[:, 1:3] = size[:, 1:3] / cfg.TRACKLET_GTBOX_LENGTH_SCALE for j in range(len(translation)): if 0 < translation[j, 1] < 8: tracklet.add_tracklet(frame_num, size[j], translation[j], rotation[j]) resize_scale = top_image.shape[0] / rgb_image.shape[0] rgb_image = cv2.resize( rgb_image, (int(rgb_image.shape[1] * resize_scale), top_image.shape[0])) rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB) new_image = np.concatenate((top_image, rgb_image), axis=1) cv2.imwrite(os.path.join(log_dir, 'image', '%5d_image.jpg' % i), new_image) if generate_video: vid_in.writeFrame(new_image) vid_in.close() tracklet.write_tracklet() predict.dump_weigths(os.path.join(log_dir, 'pretrained_model')) if cfg.TRACKING_TIMER: predict.log_msg.write('It takes %0.2f secs for inferring the whole test dataset. \n' % \ (time_it.total_time())) print("tracklet file named tracklet_labels.xml is written successfully.") return tracklet.path
def draw_bbox_on_lidar_top(top, boxes3d, one_frame_tag): path = os.path.join(config.cfg.LOG_DIR, 'test', 'top', '%s.png' % one_frame_tag.replace('/', '_')) top_image = data.draw_top_image(top) top_image = data.draw_box3d_on_top(top_image, boxes3d, color=(0, 0, 80)) cv2.imwrite(path, top_image) print('write %s finished' % path)
def __call__(self, max_iter=1000, train_set =None, validation_set =None): sess = self.sess net = self.net with sess.as_default(): #for init model batch_size=1 validation_step=40 ckpt_save_step=200 if cfg.TRAINING_TIMER: time_it = timer() # start training here ######################################################################################### self.log_msg.write('iter | top_cls_loss reg_loss | fuse_cls_loss reg_loss total | \n') self.log_msg.write('-------------------------------------------------------------------------------------\n') for iter in range(max_iter): is_validation = False summary_it = False summary_runmeta = False print_loss = False log_this_iter = False # set fit flag if iter % validation_step == 0: summary_it,is_validation,print_loss = True,True,True # summary validation loss if (iter+1) % validation_step == 0: summary_it,print_loss = True,True # summary train loss if iter % 20 == 0: print_loss = True #print train loss if 1 and iter%300 == 0: summary_it,summary_runmeta = True,True if iter % self.iter_debug == 0 or (iter + 1) % self.iter_debug == 0: log_this_iter = True print('Summary log image') if iter % self.iter_debug == 0: is_validation =False else: is_validation =True data_set = self.validation_set if is_validation else self.train_set self.default_summary_writer = self.val_summary_writer if is_validation else self.train_summary_writer step_name = 'validation' if is_validation else 'training' # load dataset if config.cfg.USE_RNN: self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load(batch_size, shuffled=False) else: self.batch_rgb_images, self.batch_top_view, self.batch_front_view, \ self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id = \ data_set.load(batch_size, shuffled=True) # fit_iterate log init if log_this_iter: self.time_str = strftime("%Y_%m_%d_%H_%M", localtime()) self.frame_info = data_set.get_frame_info(self.frame_id)[0] self.log_subdir = step_name + '/' + self.time_str top_image = data.draw_top_image(self.batch_top_view[0]) self.top_image = self.top_image_padding(top_image) # fit t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss= \ self.fit_iteration(self.batch_rgb_images, self.batch_top_view, self.batch_front_view, self.batch_gt_labels, self.batch_gt_boxes3d, self.frame_id, is_validation =is_validation, summary_it=summary_it, summary_runmeta=summary_runmeta, log=log_this_iter) if print_loss: self.log_msg.write('%10s: | %5d %0.5f %0.5f | %0.5f %0.5f \n' % \ (step_name, self.n_global_step, t_cls_loss, t_reg_loss, f_cls_loss, f_reg_loss)) if iter%ckpt_save_step==0: self.save_weights(self.train_target) if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train %d iterations. \n' % \ (time_it.time_diff_per_n_loops(), ckpt_save_step)) self.gc() self.n_global_step += 1 if cfg.TRAINING_TIMER: self.log_msg.write('It takes %0.2f secs to train the dataset. \n' % \ (time_it.total_time())) self.save_progress()