def conf_thresholding(data_dir, save_dir, score_threshold): # collect all trajectories tracker_id_score = dict() eval_dir = os.path.join(data_dir, 'data') seq_list, num_seq = load_list_from_folder(eval_dir) for seq_file in seq_list: seq_data, num_line = load_txt_file(seq_file) for data_line in seq_data: data_split = data_line.split(' ') score_tmp = float(data_split[-1]) id_tmp = int(data_split[1]) if id_tmp not in tracker_id_score.keys(): tracker_id_score[id_tmp] = list() tracker_id_score[id_tmp].append(score_tmp) # collect the ID to remove based on the confidence to_delete_id = list() for track_id, score_list in tracker_id_score.items(): average_score = sum(score_list) / float(len(score_list)) if average_score < score_threshold: to_delete_id.append(track_id) # remove the ID in the data folder for tracking evaluation save_dir_tmp = os.path.join(save_dir, 'data') mkdir_if_missing(save_dir_tmp) for seq_file in seq_list: seq_name = fileparts(seq_file)[1] seq_file_save = os.path.join(save_dir_tmp, seq_name + '.txt') seq_file_save = open(seq_file_save, 'w') seq_data, num_line = load_txt_file(seq_file) for data_line in seq_data: data_split = data_line.split(' ') id_tmp = int(float(data_split[1])) if id_tmp not in to_delete_id: seq_file_save.write(data_line + '\n') seq_file_save.close() # remove the ID in the trk with id folder for detection evaluation and tracking visualization trk_id_dir = os.path.join(data_dir, 'trk_withid') seq_dir_list, num_seq = load_list_from_folder(trk_id_dir) save_dir_tmp = os.path.join(save_dir, 'trk_withid') for seq_dir in seq_dir_list: frame_list, num_frame = load_list_from_folder(seq_dir) seq_name = fileparts(seq_dir)[1] save_frame_dir = os.path.join(save_dir_tmp, seq_name) mkdir_if_missing(save_frame_dir) for frame in frame_list: frame_index = fileparts(frame)[1] frame_file_save = os.path.join(save_frame_dir, frame_index + '.txt') frame_file_save = open(frame_file_save, 'w') frame_data, num_line = load_txt_file(frame) for data_line in frame_data: data_split = data_line.split(' ') id_tmp = int(data_split[-1]) if id_tmp not in to_delete_id: frame_file_save.write(data_line + '\n') frame_file_save.close()
def sweep_data(self): ''' sweep the data and return the dictionary of ground truth infomation for every images outputs: images_dict: a dictionary of info, keys are the image ids, values are a dictionary {'width':, 'height':, 'ids': a list of object ids contained} ''' print('loading annotations into memory from %s...' % self.anno_dir) anno_list, num_anno = load_list_from_folder(self.anno_dir, ext_filter=['.json'], recursive=True, depth=2) print('number of annotations has been loaded: %d' % num_anno) images_dict = {} for anno_tmp in anno_list: _, filename, _ = fileparts(anno_tmp) image_id = filename.split('_gt')[0] img_shape, class_ids = self.anno2mask(anno_tmp, run_fast=True) class_ids = list(set(class_ids)) image_data = { 'width': img_shape[0], 'height': img_shape[1], 'ids': class_ids } images_dict[image_id] = image_data return images_dict
def vis(result_sha, data_root, result_root): def show_image_with_boxes(img, objects_res, object_gt, calib, save_path, height_threshold=0): img2 = np.copy(img) for obj in objects_res: box3d_pts_2d, _ = compute_box_3d(obj, calib.P) color_tmp = tuple([int(tmp * 255) for tmp in colors[obj.id % max_color]]) img2 = draw_projected_box3d(img2, box3d_pts_2d, color=color_tmp) text = 'ID: %d' % obj.id if box3d_pts_2d is not None: img2 = cv2.putText(img2, text, (int(box3d_pts_2d[4, 0]), int(box3d_pts_2d[4, 1]) - 8), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color=color_tmp) img = Image.fromarray(img2) img = img.resize((width, height)) img.save(save_path) for seq in seq_list: image_dir = os.path.join(data_root, 'image_02/%s' % seq) calib_file = os.path.join(data_root, 'calib/%s.txt' % seq) result_dir = os.path.join(result_root, '%s/trk_withid/%s' % (result_sha, seq)) save_3d_bbox_dir = os.path.join(result_dir, '../../trk_image_vis/%s' % seq); mkdir_if_missing(save_3d_bbox_dir) # load the list images_list, num_images = load_list_from_folder(image_dir) print('number of images to visualize is %d' % num_images) start_count = 0 for count in range(start_count, num_images): image_tmp = images_list[count] if not is_path_exists(image_tmp): count += 1 continue image_index = int(fileparts(image_tmp)[1]) image_tmp = np.array(Image.open(image_tmp)) img_height, img_width, img_channel = image_tmp.shape result_tmp = os.path.join(result_dir, '%06d.txt'%image_index) # load the result if not is_path_exists(result_tmp): object_res = [] else: object_res = read_label(result_tmp) print('processing index: %d, %d/%d, results from %s' % (image_index, count+1, num_images, result_tmp)) calib_tmp = Calibration(calib_file) # load the calibration object_res_filtered = [] for object_tmp in object_res: if object_tmp.type not in type_whitelist: continue if hasattr(object_tmp, 'score'): if object_tmp.score < score_threshold: continue center = object_tmp.t object_res_filtered.append(object_tmp) num_instances = len(object_res_filtered) save_image_with_3dbbox_gt_path = os.path.join(save_3d_bbox_dir, '%06d.jpg' % (image_index)) show_image_with_boxes(image_tmp, object_res_filtered, [], calib_tmp, save_path=save_image_with_3dbbox_gt_path) print('number of objects to plot is %d' % (num_instances)) count += 1
def test_simple(params): """Test function.""" left = tf.placeholder(tf.float32, [2, args.input_height, args.input_width, 3]) model = MonodepthModel(params, "test", left, None) # SESSION config = tf.ConfigProto(allow_soft_placement=True) sess = tf.Session(config=config) # SAVER train_saver = tf.train.Saver() # INIT sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) coordinator = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coordinator) # RESTORE restore_path = checkpoint_path.split(".")[0] train_saver.restore(sess, restore_path) image_list, num_images = load_list_from_folder(images_dir) print('number of images loaded is %d' % num_images) for image_file_tmp in image_list: input_image = scipy.misc.imread(image_file_tmp, mode="RGB") original_height, original_width, num_channels = input_image.shape input_image = scipy.misc.imresize( input_image, [args.input_height, args.input_width], interp='lanczos') input_image = input_image.astype(np.float32) / 255 input_images = np.stack((input_image, np.fliplr(input_image)), 0) _, filename, ext = fileparts(image_file_tmp) print('processing frame %s' % filename) disp = sess.run(model.disp_left_est[0], feed_dict={left: input_images}) disp_pp = post_process_disparity(disp.squeeze()).astype(np.float32) # output_directory = os.path.dirname(args.image_path) # output_name = os.path.splitext(os.path.basename(args.image_path))[0] data_file_tmp = os.path.join(depth_save_dir, filename + '.npy') np.save(data_file_tmp, disp_pp) # np.save(os.path.join(output_directory, "{}_disp.npy".format(output_name)), disp_pp) disp_to_img = scipy.misc.imresize(disp_pp.squeeze(), [original_height, original_width]) save_file_tmp = os.path.join(depth_img_save_dir, filename + '.png') plt.imsave(save_file_tmp, disp_to_img, cmap='plasma') print('done!')
def epoch(self, model, epoch): print_log("Starting validation...", log=self.log_file) count = 0 validator_function = model.validator_function() for i_batch, (sample_batched, filename_batch) in enumerate(self.validationdataloader): with torch.no_grad(): input = Variable(sample_batched['temporalvolume']) labels = sample_batched['label'] if (self.usecudnn): input = input.cuda(self.gpuid) labels = labels.cuda(self.gpuid) # num_batch x 1 outputs = model( input ) # num_batch x 500 for temp-conv num_batch x 29 x 500 count_tmp, predict_index_list = validator_function( outputs, labels) count += count_tmp for batch_index in range(self.batchsize): filename_tmp = filename_batch[batch_index] _, filename_tmp, _ = fileparts(filename_tmp) filename_tmp = filename_tmp.split('_')[0] prediction_tmp = self.validationdataset.label_list[ predict_index_list[batch_index]] print_log( 'Evaluation: val set, batch index %d/%d, filename: %s, prediction: %s' % (batch_index + 1, self.batchsize, filename_tmp, prediction_tmp), log=self.log_file) print_log( 'Evaluation: val set, batch %d/%d, correct so far %d/%d' % (i_batch + 1, self.num_batches, count, self.batchsize * (i_batch + 1)), log=self.log_file) accuracy = count / len(self.validationdataset) accu_savepath = os.path.join(self.savedir, 'accuracy_epoch%03d.txt' % epoch) print_log('saving the accuracy file to %s' % accu_savepath, log=self.log_file) with open(accu_savepath, "a") as outputfile: outputfile.write( "\ncorrect count: {}, total count: {} accuracy: {}".format( count, len(self.validationdataset), accuracy))
def visualization(self, image_index, save_dir): ''' visualize the images loaded into the dataset ''' image_path = self.image_info[image_index]['path'] _, filename, _ = fileparts(image_path) image = self.load_image(image_index) masks, class_ids = self.load_mask(image_index) bbox = bboxes_from_mask(masks) # TLBR format, N x 4 fig, _ = visualize_image_with_bbox_mask(image, boxes=bbox, masks=masks, class_ids=class_ids, class_names=['BG'] + cityscape_class_names) save_path_tmp = os.path.join(save_dir, filename + '.jpg') save_vis_close_helper(fig=fig, transparent=False, save_path=save_path_tmp)
def __init__(self, options): self.batchsize = options["input"]["batchsize"] self.trainingdataset = LipreadingDataset(options["general"]["dataset"], "train") self.trainingdataloader = DataLoader( self.trainingdataset, batch_size=self.batchsize, shuffle=options["training"]["shuffle"], num_workers=options["input"]["numworkers"], drop_last=True) self.usecudnn = options["general"]["usecudnn"] self.statsfrequency = options["training"]["statsfrequency"] self.gpuid = options["general"]["gpuid"] self.learningrate = options["training"]["learningrate"] # self.modelType = options["training"]["learningrate"] self.weightdecay = options["training"]["weightdecay"] self.momentum = options["training"]["momentum"] self.log_file = options["general"]["logfile"] self.modelsavedir = options["general"]["modelsavedir"] _, self.time_str, _ = fileparts(self.modelsavedir) print_log('loaded training dataset with %d data' % len(self.trainingdataset), log=options["general"]["logfile"])
def __getitem__(self, image_index): # Get GT bounding boxes and masks for image. image_path = self.dataset.image_info[image_index]['path'] _, filename, _ = fileparts(image_path) # print(image_index) if image_index == 155: self.skip = False if self.skip: return [], [], [], [], [], [], [], image_index, filename image_id = self.image_ids[image_index] # print(image_index) # print(filename) image, image_metas, gt_class_ids, gt_boxes, gt_masks = load_image_gt(self.dataset, self.config, image_id, augment=self.augment, use_mini_mask=self.config.USE_MINI_MASK) # Skip images that have no instances. This can happen in cases # where we train on a subset of classes and the image doesn't # have any of the classes we care about. if not np.any(gt_class_ids > 0): return [], [], [], [], [], [], [], image_index, filename rpn_match, rpn_bbox = build_rpn_targets(image.shape, self.anchors_, gt_class_ids, gt_boxes, self.config) # find the matches for every anchor and the deltas # If more instances than fits in the array, sub-sample from them. if gt_boxes.shape[0] > self.config.MAX_GT_INSTANCES: ids = np.random.choice(np.arange(gt_boxes.shape[0]), self.config.MAX_GT_INSTANCES, replace=False) gt_class_ids = gt_class_ids[ids] gt_boxes = gt_boxes[ids] gt_masks = gt_masks[:, :, ids] rpn_match = rpn_match[:, np.newaxis] images = mold_image(image.astype(np.float32), self.config) # Add to batch # Convert images = torch.from_numpy(images.transpose(2, 0, 1)).float() image_metas = torch.from_numpy(image_metas) rpn_match, rpn_bbox = torch.from_numpy(rpn_match), torch.from_numpy(rpn_bbox).float() gt_class_ids, gt_boxes, gt_masks = torch.from_numpy(gt_class_ids), torch.from_numpy(gt_boxes).float(), torch.from_numpy(gt_masks.astype(int).transpose(2, 0, 1)).float() return images, image_metas, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, image_index, filename
if __name__ == '__main__': if len(sys.argv) != 2: print('Usage: python main.py result_sha(e.g., pointrcnn_Car_test)') sys.exit(1) result_sha = sys.argv[1] save_root = './results' det_id2str = {1:'Pedestrian', 2:'Car', 3:'Cyclist'} seq_file_list, num_seq = load_list_from_folder(os.path.join('data/KITTI', result_sha)) total_time, total_frames = 0.0, 0 save_dir = os.path.join(save_root, result_sha); mkdir_if_missing(save_dir) eval_dir = os.path.join(save_dir, 'data'); mkdir_if_missing(eval_dir) seq_count = 0 for seq_file in seq_file_list: _, seq_name, _ = fileparts(seq_file) eval_file = os.path.join(eval_dir, seq_name + '.txt'); eval_file = open(eval_file, 'w') save_trk_dir = os.path.join(save_dir, 'trk_withid', seq_name); mkdir_if_missing(save_trk_dir) mot_tracker = AB3DMOT() seq_dets = np.loadtxt(seq_file, delimiter=',') # load detections, N x 15 # print(seq_file) # print(seq_dets) print(seq_dets.shape) # print(len(seq_dets.shape)) break # if no detection in a sequence if len(seq_dets.shape) == 1: seq_dets = np.expand_dims(seq_dets, axis=0) if seq_dets.shape[1] == 0: eval_file.close()
def generate_hdf5(data_src, save_dir, data_name='data', batch_size=1, ext_filter='png', label_src1=None, label_name1='label', label_preprocess_function1=identity, label_range1=None, label_src2=None, label_name2='label2', label_preprocess_function2=identity, label_range2=None, debug=True, vis=False): ''' # this function creates data in hdf5 format from a image path # input parameter # data_src: source of image data, which can be a list of image path, a txt file contains a list of image path, a folder contains a set of images, a list of numpy array image data # label_src: source of label data, which can be none, a file contains a set of labels, a dictionary of labels, a 1-d numpy array data, a list of label data # save_dir: where to store the hdf5 data # batch_size: how many image to store in a single hdf file # ext_filder: what format of data to use for generating hdf5 data ''' # parse input assert is_path_exists_or_creatable( save_dir), 'save path should be a folder to save all hdf5 files' mkdir_if_missing(save_dir) assert isstring( data_name), 'dataset name is not correct' # name for hdf5 data # convert data source to a list of numpy array image data if isfolder(data_src): print 'data is loading from %s with extension .%s' % (data_src, ext_filter) filelist, num_data = load_list_from_folder(data_src, ext_filter=ext_filter) datalist = None elif isfile(data_src): print 'data is loading from %s with extension .%s' % (data_src, ext_filter) filelist, num_data = load_list_from_file(data_src) datalist = None elif islist(data_src): if debug: assert all( isimage(data_tmp) for data_tmp in data_src ), 'input data source is not a list of numpy array image data' datalist = data_src num_data = len(datalist) filelist = None else: assert False, 'data source format is not correct.' if debug: assert (datalist is None and filelist is not None) or ( filelist is None and datalist is not None), 'data is not correct' if datalist is not None: assert len(datalist) == num_data, 'number of data is not equal' if filelist is not None: assert len(filelist) == num_data, 'number of data is not equal' # convert label source to a list of numpy array label if label_src1 is None: labeldict1 = None labellist1 = None elif isfile(label_src1): assert is_path_exists(label_src1), 'file not found' _, _, ext = fileparts(label_src1) assert ext == '.json', 'only json extension is supported' labeldict1 = json.load(label_src1) num_label1 = len(labeldict1) assert num_data == num_label1, 'number of data and label is not equal.' labellist1 = None elif isdict(label_src1): labeldict1 = label_src1 labellist1 = None elif isnparray(label_src1): if debug: assert label_src1.ndim == 1, 'only 1-d label is supported' labeldict1 = None labellist1 = label_src1 elif islist(label_src1): if debug: assert all( np.array(label_tmp).size == 1 for label_tmp in label_src1), 'only 1-d label is supported' labellist1 = label_src1 labeldict1 = None else: assert False, 'label source format is not correct.' assert isfunction(label_preprocess_function1 ), 'label preprocess function is not correct.' # convert label source to a list of numpy array label if label_src2 is None: labeldict2 = None labellist2 = None elif isfile(label_src2): assert is_path_exists(label_src2), 'file not found' _, _, ext = fileparts(label_src2) assert ext == '.json', 'only json extension is supported' labeldict2 = json.load(label_src2) num_label2 = len(labeldict2) assert num_data == num_label2, 'number of data and label is not equal.' labellist2 = None elif isdict(label_src2): labeldict2 = label_src2 labellist2 = None elif isnparray(label_src2): if debug: assert label_src2.ndim == 1, 'only 1-d label is supported' labeldict2 = None labellist2 = label_src2 elif islist(label_src2): if debug: assert all( np.array(label_tmp).size == 1 for label_tmp in label_src2), 'only 1-d label is supported' labellist2 = label_src2 labeldict2 = None else: assert False, 'label source format is not correct.' assert isfunction(label_preprocess_function2 ), 'label preprocess function is not correct.' # warm up if datalist is not None: size_data = datalist[0].shape else: size_data = imread(filelist[0]).shape if labeldict1 is not None: if debug: assert isstring(label_name1), 'label name is not correct' labels1 = np.zeros((batch_size, 1), dtype='float32') # label_value1 = [float(label_tmp_char) for label_tmp_char in labeldict1.values()] # label_range1 = np.array([min(label_value1), max(label_value1)]) if labellist1 is not None: labels1 = np.zeros((batch_size, 1), dtype='float32') # label_range1 = [np.min(labellist1), np.max(labellist1)] if label_src1 is not None and debug: assert label_range1 is not None, 'label range is not correct' assert (labeldict1 is not None and labellist1 is None) or ( labellist1 is not None and labeldict1 is None), 'label is not correct' if labeldict2 is not None: if debug: assert isstring(label_name2), 'label name is not correct' labels2 = np.zeros((batch_size, 1), dtype='float32') # label_value2 = [float(label_tmp_char) for label_tmp_char in labeldict2.values()] # label_range2 = np.array([min(label_value2), max(label_value2)]) if labellist2 is not None: labels2 = np.zeros((batch_size, 1), dtype='float32') # label_range2 = [np.min(labellist2), np.max(labellist2)] if label_src2 is not None and debug: assert label_range2 is not None, 'label range is not correct' assert (labeldict2 is not None and labellist2 is None) or ( labellist2 is not None and labeldict2 is None), 'label is not correct' # start generating count_hdf = 1 # count number of hdf5 file clock = Timer() datalist_batch = list() for i in xrange(num_data): clock.tic() if filelist is not None: imagefile = filelist[i] _, name, _ = fileparts(imagefile) img = imread(imagefile).astype('float32') max_value = np.max(img) if max_value > 1 and max_value <= 255: img = img / 255.0 # [rows,col,channel,numbers], scale the image data to (0, 1) if debug: min_value = np.min(img) assert min_value >= 0 and min_value <= 1, 'data is not in [0, 1]' if datalist is not None: img = datalist[i] if debug: assert size_data == img.shape datalist_batch.append(img) # process label if labeldict1 is not None: if debug: assert len(filelist) == len( labeldict1), 'file list is not equal to label dictionary' labels1[i % batch_size, 0] = float(labeldict1[name]) if labellist1 is not None: labels1[i % batch_size, 0] = float(labellist1[i]) if labeldict2 is not None: if debug: assert len(filelist) == len( labeldict2), 'file list is not equal to label dictionary' labels2[i % batch_size, 0] = float(labeldict2[name]) if labellist2 is not None: labels2[i % batch_size, 0] = float(labellist2[i]) # save to hdf5 if i % batch_size == 0: data = preprocess_image_caffe( datalist_batch, debug=debug, vis=vis ) # swap channel, transfer from list of HxWxC to NxCxHxW # write to hdf5 format if filelist is not None: save_path = os.path.join(save_dir, '%s.hdf5' % name) else: save_path = os.path.join(save_dir, 'image_%010d.hdf5' % count_hdf) h5f = h5py.File(save_path, 'w') h5f.create_dataset(data_name, data=data, dtype='float32') if (labeldict1 is not None) or (labellist1 is not None): # print(labels1) labels1 = label_preprocess_function1(data=labels1, data_range=label_range1, debug=debug) # print(labels1) h5f.create_dataset(label_name1, data=labels1, dtype='float32') labels1 = np.zeros((batch_size, 1), dtype='float32') if (labeldict2 is not None) or (labellist2 is not None): labels2 = label_preprocess_function2(data=labels2, data_range=label_range2, debug=debug) h5f.create_dataset(label_name2, data=labels2, dtype='float32') labels2 = np.zeros((batch_size, 1), dtype='float32') h5f.close() count_hdf = count_hdf + 1 del datalist_batch[:] if debug: assert len(datalist_batch) == 0, 'list has not been cleared' average_time = clock.toc() print( 'saving to %s: %d/%d, average time:%.3f, elapsed time:%s, estimated time remaining:%s' % (save_path, i + 1, num_data, average_time, convert_secs2time(average_time * i), convert_secs2time(average_time * (num_data - i)))) return count_hdf - 1, num_data
def evaluate(args, loader, generator, num_samples, path): # ade_outer, fde_outer = [], [] ade_all, fde_all = AverageMeter(), AverageMeter() total_obj = 0 pred_len = args.pred_len dataset_name = args.dataset_name obj_class = dataset_name.split('_')[1][:3] save_dir, _, _ = fileparts(path) save_dir = os.path.join(save_dir, 'results_%s' % get_timestring()) mkdir_if_missing(save_dir) result_file_single = os.path.join(save_dir, 'results.json') result_dict = dict() with torch.no_grad(): for batch in loader: batch = [tensor.cuda() for tensor in batch] (obs_traj, pred_traj_gt, obs_traj_rel, pred_traj_gt_rel, non_linear_ped, loss_mask, seq_start_end, id_frame) = batch # obs_traj frames x objects x 2 # pred_traj_gt frames x objects x 2 # seq_Start_end start, end of ped index in each timestamp, used for pooling at every timestamp # id_frame 2frames x objects x 3 # loss_mask objects x 2frames num_obs = obs_traj.size(0) num_objects = obs_traj.size(1) id_frame_pred = id_frame[num_obs:] # frames x obj x 3 loss_mask_pred = loss_mask[:, num_obs:] # objects x seq_len ade, fde = [], [] for sample_index in range(num_samples): pred_traj_fake_rel = generator(obs_traj, obs_traj_rel, seq_start_end) pred_traj_fake = relative_to_abs( pred_traj_fake_rel, obs_traj[-1]) # frames x objects x 2 # save results for object_index in range(num_objects): id_frame_tmp = id_frame_pred[:, object_index, :] frame = int(id_frame_tmp[0, 0].item()) # seqname should be the same across frames seq = np.unique(id_frame_tmp[:, -1].cpu().clone().numpy()) assert len(seq) == 1, 'error' seqname = int2seqname(seq[0]) # AIODrive only # seqname should be the same across frames ID = np.unique(id_frame_tmp[:, 1].cpu().clone().numpy()) assert len(ID) == 1, 'error' ID = int(ID[0]) # saving to individual frames final_results = torch.cat([ id_frame_tmp[:, :2], pred_traj_fake[:, object_index, :] ], axis=-1).cpu().clone().numpy() save_path = os.path.join( save_dir, seqname, 'frame_%06d' % (frame), 'sample_%03d' % sample_index + '.txt') mkdir_if_missing(save_path) with open(save_path, 'a') as f: np.savetxt(f, final_results, fmt="%.3f") # saving to a single file, result format # {seqname1: {frame1: {sample1: {ID1: {state: N x 2, prob: 1}}}, seqname2, ...} if seqname not in result_dict.keys(): result_dict[seqname] = dict() if frame not in result_dict[seqname].keys(): result_dict[seqname][frame] = dict() if sample_index not in result_dict[seqname][frame].keys(): result_dict[seqname][frame][sample_index] = dict() if ID not in result_dict[seqname][frame][ sample_index].keys(): result_dict[seqname][frame][sample_index][ID] = dict() result_dict[seqname][frame][sample_index][ ID]['state'] = pred_traj_fake[:, object_index, :].cpu( ).clone().numpy().tolist() result_dict[seqname][frame][sample_index][ID]['prob'] = 1.0 # compute ADE ade_tmp = displacement_error( pred_traj_fake, pred_traj_gt, mode='raw', mask=loss_mask_pred ) # list of ade for each object in the batch ade.append(ade_tmp) # list of error for all samples # select the right last timestamp for FDE computation, i.e., not select the last frame if masked out pred_traj_last = [] gt_traj_last = [] for obj_tmp in range(num_objects): loss_mask_tmp = loss_mask_pred[obj_tmp] # seq_len good_index = torch.nonzero(loss_mask_tmp) if torch.nonzero(loss_mask_tmp).size(0) == 0: pred_traj_last.append(torch.zeros(2).cuda() / 0) gt_traj_last.append(torch.zeros(2).cuda() / 0) else: last_index = torch.max(good_index) pred_traj_last.append(pred_traj_fake[last_index, obj_tmp, :]) gt_traj_last.append(pred_traj_gt[last_index, obj_tmp, :]) gt_traj_last = torch.stack(gt_traj_last, dim=0) # num_obj x 2 pred_traj_last = torch.stack(pred_traj_last, dim=0) # num_obj x 2 # compute FDE fde_tmp = final_displacement_error(pred_traj_last, gt_traj_last, mode='raw') fde.append(fde_tmp) # list of error for all samples # select the one sample with the minimum errors, remove nan num_invalid = torch.sum(torch.isnan(ade_tmp)) num_valid = pred_traj_gt.size(1) - num_invalid total_obj += num_valid # only add No.obj if it is valid, not all future frames are padded ade_ave, num_obj = best_of_K(ade, seq_start_end, err_type='ADE') fde_ave, num_obj = best_of_K(fde, seq_start_end, err_type='FDE') ade_all.update(ade_ave, n=num_obj) fde_all.update(fde_ave, n=num_obj) actual_len = pred_len * args.skip final_dict = {actual_len: {obj_class: result_dict}} with open(result_file_single, 'w') as outfile: json.dump(final_dict, outfile) return ade_all.avg, fde_all.avg
# print(test) # zxc frame1 = load_image(image_list[0]) prvs = rgb2gray(frame1) # uint8 # print(prvs) hsv = np.zeros_like(frame1) hsv[..., 1] = 255 image_index = 1 while(1): image_path = image_list[image_index] _, filename, _ = fileparts(image_path) frame2 = load_image(image_path) next = rgb2gray(frame2) # save_image(next, '/home/xinshuo/aa.jpg') # zxc # cv2.imshow('asd', next) # k = cv2.waitKey(30) # zxc # ret, frame2 = cap.read() # next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY) flow_uv = cv2.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) rgb = flow_to_color(flow_uv, convert_to_bgr=False)
assert False, 'error' model = MaskRCNN(model_dir=save_dir, config=config) # Create model object. if config.GPU_COUNT: model = model.cuda() model.load_weights(model_path) # Load weights ##--------------------------------- Testing ----------------------------------## image_list, num_list = load_list_from_folder(images_dir, ext_filter=['.png', '.jpg'], depth=2) print_log('testing results on %d images' % num_list, log=log_file) count = 1 timer = Timer() timer.tic() for image_file_tmp in image_list: parent_dir, filename, _ = fileparts(image_file_tmp) video_dir = parent_dir.split('/')[-1] # print(video_dir) # zxc image = load_image(image_file_tmp) results = model.detect([image]) # inference, results is a dictionary if len(results) == 0: count += 1 print_log('Mask-RCNN demo: testing %d/%d, no detected results!!!!!' % (count, num_list), log=log_file) continue # visualize and save results r = results[0] # results from the first image
def vis(cam_transform, data_root, result_root): def show_image_with_boxes(img, objects_res, object_gt, calib, save_path, height_threshold=0): img2 = np.copy(img) for obj in objects_res: box3d_pts_2d, _ = compute_box_3d(obj, calib.P) color_tmp = tuple( [int(tmp * 255) for tmp in colors[obj.id % max_color]]) img2 = draw_projected_box3d(img2, box3d_pts_2d, color=color_tmp) text = 'ID: %d' % obj.id if box3d_pts_2d is not None: img2 = cv2.putText( img2, text, (int(box3d_pts_2d[4, 0]), int(box3d_pts_2d[4, 1]) - 8), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color=color_tmp) img = Image.fromarray(img2) img = img.resize((width, height)) img = img.convert('RGB') img.save(save_path) for seq in seq_list: #image_dir = os.path.join(data_root, 'image_02/%s' % seq) #calib_file = os.path.join(data_root, 'calib/%s.txt' % seq) #result_dir = os.path.join(result_root, '%s/trk_withid/%s' % (result_sha, seq)) #save_3d_bbox_dir = os.path.join(result_dir, '../../trk_image_vis/%s' % seq); mkdir_if_missing(save_3d_bbox_dir) image_dir = '/home/ubuntu/xwp/datasets/multi_view_dataset/new/cam_sample/image_2' result_dir = '/home/ubuntu/xwp/datasets/multi_view_dataset/new/fuse_test/cam9+cam21/tracking_results/trk_withid/0000' calib_file = '/home/ubuntu/xwp/datasets/multi_view_dataset/346/calib/000000.txt' save_3d_bbox_dir = '/home/ubuntu/xwp/datasets/multi_view_dataset/new/fuse_test/cam9+cam21/trk_image_vis' mkdir_if_missing(save_3d_bbox_dir) # load the list images_list, num_images = load_list_from_folder(image_dir) print('number of images to visualize is %d' % num_images) cam_id = 'cam9' start_count = 8945 end_count = 8950 min_index = 8900 for count in range(start_count, end_count): image_tmp = images_list[count] if not is_path_exists(image_tmp): count += 1 continue image_index = int(fileparts(image_tmp)[1]) image_tmp = np.array(Image.open(image_tmp)) img_height, img_width, img_channel = image_tmp.shape result_tmp = os.path.join( result_dir, '%06d.txt' % (image_index - min_index)) # load the result if not is_path_exists(result_tmp): object_res = [] else: object_res = read_label(result_tmp) print('processing index: %d, %d/%d, results from %s' % (image_index, count + 1, num_images, result_tmp)) calib_tmp = Calibration(calib_file) # load the calibration object_res_filtered = [] cam_trans = cam_transform[cam_id] for object_tmp in object_res: if object_tmp.type not in type_whitelist: continue if hasattr(object_tmp, 'score'): if object_tmp.score < score_threshold: continue center = object_tmp.t #transform cord_p = np.zeros((1, 4)) cord_p[0][0] = object_tmp.t[2] cord_p[0][1] = object_tmp.t[0] cord_p[0][2] = -object_tmp.t[1] cord_p[0][3] = 1 rotation_y = object_tmp.ry * 180 / np.pi cam_matrix = cu.ClientSideBoundingBoxes.get_matrix(cam_trans) cam_to_world = np.dot(cam_matrix, np.transpose(cord_p)) ry_cam2world = cu.ry_filter_a( rotation_y - 90 + cam_trans.rotation.yaw) * np.pi / 180 object_tmp.t = [ cam_to_world[1][0], -cam_to_world[2][0], cam_to_world[0][0] ] object_tmp.ry = cu.ry_filter(ry_cam2world) #end object_res_filtered.append(object_tmp) num_instances = len(object_res_filtered) save_image_with_3dbbox_gt_path = os.path.join( save_3d_bbox_dir, '%06d.jpg' % (image_index)) show_image_with_boxes(image_tmp, object_res_filtered, [], calib_tmp, save_path=save_image_with_3dbbox_gt_path) print('number of objects to plot is %d' % (num_instances)) count += 1
def __init__(self, data_dir, obs_len=8, pred_len=12, skip=1, threshold=0.002, min_ped=0, delim='\t', \ phase='training', split='test'): """ Args: - data_dir: Directory containing dataset files in the format <frame_id> <ped_id> <x> <y> - obs_len: Number of time-steps in input trajectories - pred_len: Number of time-steps in output trajectories - skip: Number of frames to skip while making the dataset - threshold: Minimum error to be considered for non linear traj when using a linear predictor - min_ped: Minimum number of pedestrians that should be in a seqeunce - delim: Delimiter in the dataset files """ super(TrajectoryDataset, self).__init__() self.data_dir = data_dir self.obs_len, self.pred_len = obs_len, pred_len self.seq_len = self.obs_len + self.pred_len self.delim, self.skip = delim, skip all_files, _ = load_list_from_folder(self.data_dir) num_peds_in_seq = [] seq_list = [] seq_list_rel = [] loss_mask_list = [] non_linear_ped = [] seq_id_list = [] for path in all_files: print_str = 'load %s\r' % path sys.stdout.write(print_str) sys.stdout.flush() _, seq_name, _ = fileparts(path) data = read_file(path, delim) # as testing files only contains past, so add more windows if split == 'test': min_frame, max_frame = 0, 999 num_windows = int(max_frame - min_frame + 1 - skip * (self.seq_len - 1)) num_windows += (self.pred_len - 1) * skip + 1 else: frames = np.unique(data[:, 0]).tolist() min_frame, max_frame = frames[0], frames[-1] num_windows = int(max_frame - min_frame + 1 - skip * (self.seq_len - 1) ) # include all frames for past and future # loop through every windows for window_index in range(num_windows): start_frame = int(window_index + min_frame) end_frame = int(start_frame + self.seq_len * skip) # right-open, not including this frame # reduce window during testing, only evaluate every N windows if phase == 'testing': check_pass = check_eval_windows(start_frame + self.obs_len * skip, self.obs_len * skip, self.pred_len * skip, split=split) if not check_pass: continue # get data in current window curr_seq_data = [] for frame in range(start_frame, end_frame, skip): curr_seq_data.append(data[frame == data[:, 0], :]) curr_seq_data = np.concatenate(curr_seq_data, axis=0) # initialize data peds_in_curr_seq = np.unique(curr_seq_data[:, 1]) curr_seq_rel = np.zeros( (len(peds_in_curr_seq), 2, self.seq_len)) # objects x 2 x seq_len curr_seq = np.zeros((len(peds_in_curr_seq), 2, self.seq_len)) curr_loss_mask = np.zeros( (len(peds_in_curr_seq), self.seq_len)) # objects x seq_len id_frame_list = np.zeros( (len(peds_in_curr_seq), 3, self.seq_len)) # objects x 2 x seq_len id_frame_list.fill(-1) num_peds_considered = 0 _non_linear_ped = [] # loop through every object in this window for _, ped_id in enumerate(peds_in_curr_seq): curr_ped_seq = curr_seq_data[ curr_seq_data[:, 1] == ped_id, :] # frames x 4 (frames can less than seqlen) pad_front = int(curr_ped_seq[0, 0]) - start_frame pad_end = int(curr_ped_seq[-1, 0]) - start_frame + skip assert pad_end % skip == 0, 'error' frame_existing = curr_ped_seq[:, 0].tolist() # pad front and back data to make the trajectory complete if pad_end - pad_front != self.seq_len * skip: # pad end to_be_paded_end = int(self.seq_len - pad_end / skip) pad_end_seq = np.expand_dims(curr_ped_seq[-1, :], axis=0) pad_end_seq = np.repeat(pad_end_seq, to_be_paded_end, axis=0) frame_offset = np.zeros((to_be_paded_end, 4), dtype='float32') frame_offset[:, 0] = np.array( range(1, to_be_paded_end + 1)) pad_end_seq += frame_offset * skip # shift first columns for frame curr_ped_seq = np.concatenate( (curr_ped_seq, pad_end_seq), axis=0) # pad front to_be_paded_front = int(pad_front / skip) pad_front_seq = np.expand_dims(curr_ped_seq[0, :], axis=0) pad_front_seq = np.repeat(pad_front_seq, to_be_paded_front, axis=0) frame_offset = np.zeros((to_be_paded_front, 4), dtype='float32') frame_offset[:, 0] = np.array( range(-to_be_paded_front, 0)) pad_front_seq += frame_offset * skip curr_ped_seq = np.concatenate( (pad_front_seq, curr_ped_seq), axis=0) # set pad front and end to correct values pad_front = 0 pad_end = self.seq_len * skip # add edge case when the object reappears at a bad frame # in other words, missing intermediate frame if curr_ped_seq.shape[0] != (pad_end - pad_front) / skip: frame_all = list( range(int(curr_ped_seq[0, 0]), int(curr_ped_seq[-1, 0]) + skip, skip)) frame_missing, _ = remove_list_from_list( frame_all, curr_ped_seq[:, 0].tolist()) # pad all missing frames with zeros pad_seq = np.expand_dims(curr_ped_seq[-1, :], axis=0) pad_seq = np.repeat(pad_seq, len(frame_missing), axis=0) pad_seq.fill(0) pad_seq[:, 0] = np.array(frame_missing) pad_seq[:, 1] = ped_id # fill ID curr_ped_seq = np.concatenate((curr_ped_seq, pad_seq), axis=0) curr_ped_seq = curr_ped_seq[np.argsort( curr_ped_seq[:, 0])] assert pad_front == 0, 'error' assert pad_end == self.seq_len * skip, 'error' # make sure the seq_len frames are continuous, no jumping frames start_frame_now = int(curr_ped_seq[0, 0]) if curr_ped_seq[-1, 0] != start_frame_now + (self.seq_len - 1) * skip: continue # make sure that past data has at least one frame past_frame_list = [ *range(start_frame_now, start_frame_now + self.obs_len * skip, skip) ] common = find_unique_common_from_lists(past_frame_list, frame_existing, only_com=True) if len(common) == 0: continue # make sure that future GT data has at least one frame if phase != 'testing': gt_frame_list = [ *range(start_frame_now + self.obs_len * skip, start_frame_now + self.seq_len * skip, skip) ] common = find_unique_common_from_lists(gt_frame_list, frame_existing, only_com=True) if len(common) == 0: continue # only keep the state cache_tmp = np.transpose(curr_ped_seq[:, :2]) curr_ped_seq = np.transpose( curr_ped_seq[:, 2:]) # 2 x seq_len # Make coordinates relative rel_curr_ped_seq = np.zeros(curr_ped_seq.shape) rel_curr_ped_seq[:, 1:] = curr_ped_seq[:, 1:] - curr_ped_seq[:, : -1] _idx = num_peds_considered curr_seq[_idx, :, :] = curr_ped_seq curr_seq_rel[_idx, :, :] = rel_curr_ped_seq # record seqname, frame and ID information id_frame_list[_idx, :2, :] = cache_tmp id_frame_list[_idx, 2, :] = seqname2int(seq_name) # Linear vs Non-Linear Trajectory, only fit for the future part not past part _non_linear_ped.append( poly_fit(curr_ped_seq, pred_len, threshold)) # add mask onto padded dummay data frame_exist_index = np.array([ frame_tmp - start_frame_now for frame_tmp in frame_existing ]) frame_exist_index = (frame_exist_index / skip).astype('uint8') curr_loss_mask[_idx, frame_exist_index] = 1 num_peds_considered += 1 if num_peds_considered > min_ped: non_linear_ped += _non_linear_ped num_peds_in_seq.append(num_peds_considered) loss_mask_list.append(curr_loss_mask[:num_peds_considered]) seq_list.append(curr_seq[:num_peds_considered]) seq_list_rel.append(curr_seq_rel[:num_peds_considered]) seq_id_list.append(id_frame_list[:num_peds_considered]) self.num_seq = len(seq_list) seq_list = np.concatenate(seq_list, axis=0) # objects x 2 x seq_len seq_list_rel = np.concatenate(seq_list_rel, axis=0) loss_mask_list = np.concatenate(loss_mask_list, axis=0) non_linear_ped = np.asarray(non_linear_ped) seq_id_list = np.concatenate(seq_id_list, axis=0) # Convert numpy -> Torch Tensor self.obs_traj = torch.from_numpy(seq_list[:, :, :self.obs_len]).type( torch.float) self.pred_traj = torch.from_numpy(seq_list[:, :, self.obs_len:]).type( torch.float) self.obs_traj_rel = torch.from_numpy( seq_list_rel[:, :, :self.obs_len]).type(torch.float) self.pred_traj_rel = torch.from_numpy( seq_list_rel[:, :, self.obs_len:]).type(torch.float) self.loss_mask = torch.from_numpy(loss_mask_list).type(torch.float) self.non_linear_ped = torch.from_numpy(non_linear_ped).type( torch.float) cum_start_idx = [0] + np.cumsum(num_peds_in_seq).tolist() self.seq_start_end = [ (start, end) for start, end in zip(cum_start_idx, cum_start_idx[1:]) ] self.seq_id_list = torch.from_numpy(seq_id_list).type(torch.float)
elif train_dataset == 'kitti': model_path = '/media/xinshuo/Data/models/mask_rcnn_pytorch/%s/mask_rcnn_kitti_%04d.pth' % (model_folder, epoch) else: model_path = os.path.join(root_dir, 'resnet50_imagenet.pth') # Path to trained weights from Imagenet model = MaskRCNN(model_dir=save_dir, config=config) # Create model object. if config.GPU_COUNT: model = model.cuda() model.load_weights(model_path) # Load weights print_log('load weights from %s' % model_path, log=log_file) ##--------------------------------- Build KITTI Evaluation Results ----------------------------------## id_list, num_list = load_list_from_file(split_file) print_log('KITTI Evaluation: testing results on %d images' % num_list, log=log_file) count = 1 timer = Timer(); timer.tic() # count_skipped = 0 for id_tmp in id_list: image_file_tmp = os.path.join(images_dir, id_tmp+'.png') _, filename, _ = fileparts(image_file_tmp) bbox_eval_file_tmp = os.path.join(bbox_eval_folder, filename+'.txt'); bbox_eval_file_tmp = open(bbox_eval_file_tmp, 'w') label_matching_file_tmp = os.path.join(label_bbox_match_dir, filename+'.txt') label_matching_file_tmp = open(label_matching_file_tmp, 'w') image = load_image(image_file_tmp) results = model.detect([image]) # inference, results is a dictionary elapsed = timer.toc(average=False) remaining_str = convert_secs2time((elapsed) / count * (num_list - count)) elapsed_str = convert_secs2time(elapsed) if len(results) == 0: count += 1 print_str = 'KITTI %s: trained on %s, %d epochs, %d/%d, no detected result!!' % (split, train_dataset, epoch, count, num_list) print(print_str) print_log('%s, saving to %s' % (print_str, filename), log=log_file, display=False) continue
def facial_landmark_evaluation(pred_dict_all, anno_dict, num_pts, error_threshold, normalization_ced=True, normalization_vec=False, covariance=True, display_list=None, debug=True, vis=False, save=True, save_path=None): ''' evaluate the performance of facial landmark detection parameter: pred_dict_all: a dictionary for all basline methods. Each key is the method name and the value is corresponding prediction dictionary, which keys are the image path and values are 2 x N prediction results anno_dict: a dictionary which keys are the image path and values are 2 x N annotation results num_pts: number of points vis: determine if visualizing the pck curve save: determine if saving the visualization results save_path: a directory to save all the results visualization: 1. 2d pck curve (total and point specific) for all points for all methods 2. point error vector (total and point specific) for all points and for all methods 3. mean square error return: metrics_all: a list of list to have detailed metrics over all methods ptswise_mse: a list of list to have average MSE over all key-points for all methods ''' num_methods = len(pred_dict_all) if debug: assert isdict(pred_dict_all) and num_methods > 0 and all(isdict(pred_dict) for pred_dict in pred_dict_all.values()), 'predictions result format is not correct' assert isdict(anno_dict), 'annotation result format is not correct' assert ispositiveinteger(num_pts), 'number of points is not correct' assert isscalar(error_threshold), 'error threshold is not correct' assert islogical(normalization_ced) and islogical(normalization_vec), 'normalization flag is not correct' if display_list is not None: assert len(display_list) == num_methods, 'display list is not correct %d vs %d' % (len(display_list), num_methods) num_images = len(pred_dict_all.values()[0]) if debug: assert num_images > 0, 'the predictions are empty' assert num_images == len(anno_dict), 'number of images is not equal to number of annotations: %d vs %d' % (num_images, len(anno_dict)) assert all(num_images == len(pred_dict) for pred_dict in pred_dict_all.values()), 'number of images in results from different methods are not equal' # calculate normalized mean error for each single image based on point-to-point Euclidean distance normalized by the bounding box size # calculate point error vector for each single image based on error vector normalized by the bounding box size normed_mean_error_dict = dict() normed_mean_error_pts_specific_dict = dict() normed_mean_error_pts_specific_valid_dict = dict() pts_error_vec_dict = dict() pts_error_vec_pts_specific_dict = dict() mse_error_dict_dict = dict() for method_name, pred_dict in pred_dict_all.items(): normed_mean_error_total = np.zeros((num_images, ), dtype='float32') normed_mean_error_pts_specific = np.zeros((num_images, num_pts), dtype='float32') normed_mean_error_pts_specific_valid = np.zeros((num_images, num_pts), dtype='bool') pts_error_vec = np.zeros((num_images, 2), dtype='float32') pts_error_vec_pts_specific = np.zeros((num_images, 2, num_pts), dtype='float32') mse_error_dict = dict() count = 0 count_skip_num_images = 0 # it's possible that no annotation exists on some images, than no error should be counted for those images, we count the number of those images for image_path, pts_prediction in pred_dict.items(): _, filename, _ = fileparts(image_path) pts_anno = anno_dict[filename] # 2 x N annotation pts_keep_index = range(num_pts) # to avoid list object type, do conversion here if islist(pts_anno): pts_anno = np.asarray(pts_anno) if islist(pts_prediction): pts_prediction = np.asarray(pts_prediction) if debug: assert (is2dptsarray(pts_anno) or is2dptsarray_occlusion(pts_anno)) and pts_anno.shape[1] == num_pts, 'shape of annotations is not correct (%d x %d) vs (%d x %d)' % (2, num_pts, pts_anno.shape[0], pts_anno.shape[1]) # if the annotation has 3 channels (include extra occlusion channel, we keep only the points with annotations) # occlusion: -1 -> visible but not annotated, 0 -> invisible and not annotated, 1 -> visible, we keep only visible and annotated points if pts_anno.shape[0] == 3: pts_keep_index = np.where(pts_anno[2, :] == 1)[0].tolist() if len(pts_keep_index) <= 0: # if no point is annotated in current image count_skip_num_images += 1 continue pts_anno = pts_anno[0:2, pts_keep_index] pts_prediction = pts_prediction[:, pts_keep_index] # to avoid the point location includes the score or occlusion channel, only take the first two channels here if pts_prediction.shape[0] == 3 or pts_prediction.shape[0] == 4: pts_prediction = pts_prediction[0:2, :] num_pts_tmp = len(pts_keep_index) if debug: assert pts_anno.shape[1] <= num_pts, 'number of points is not correct: %d vs %d' % (pts_anno.shape[1], num_pts) assert pts_anno.shape == pts_prediction.shape, 'shape of annotations and predictions are not the same {} vs {}'.format(print_np_shape(pts_anno, debug=debug), print_np_shape(pts_prediction, debug=debug)) # print 'number of points to keep is %d' % num_pts_tmp # calculate bbox for normalization if normalization_ced or normalization_vec: assert len(pts_keep_index) == num_pts, 'some points are not annotated. Normalization on PCK curve is not allowed.' bbox_anno = pts2bbox(pts_anno, debug=debug) # 1 x 4 bbox_TLWH = bbox_TLBR2TLWH(bbox_anno, debug=debug) # 1 x 4 bbox_size = math.sqrt(bbox_TLWH[0, 2] * bbox_TLWH[0, 3]) # scalar # calculate normalized error for all points normed_mean_error, _ = pts_euclidean(pts_prediction, pts_anno, debug=debug) # scalar if normalization_ced: normed_mean_error /= bbox_size normed_mean_error_total[count] = normed_mean_error mse_error_dict[image_path] = normed_mean_error if normed_mean_error == 0: print pts_prediction print pts_anno # calculate normalized error point specifically for pts_index in xrange(num_pts): if pts_index in pts_keep_index: # if current point not annotated in current image, just keep 0 normed_mean_error_pts_specific_valid[count, pts_index] = True else: continue pts_index_from_keep_list = pts_keep_index.index(pts_index) pts_prediction_tmp = np.reshape(pts_prediction[:, pts_index_from_keep_list], (2, 1)) pts_anno_tmp = np.reshape(pts_anno[:, pts_index_from_keep_list], (2, 1)) normed_mean_error_pts_specifc_tmp, _ = pts_euclidean(pts_prediction_tmp, pts_anno_tmp, debug=debug) if normalization_ced: normed_mean_error_pts_specifc_tmp /= bbox_size normed_mean_error_pts_specific[count, pts_index] = normed_mean_error_pts_specifc_tmp # calculate the point error vector error_vector = pts_prediction - pts_anno # 2 x num_pts_tmp if normalization_vec: error_vector /= bbox_size pts_error_vec_pts_specific[count, :, pts_keep_index] = np.transpose(error_vector) pts_error_vec[count, :] = np.sum(error_vector, axis=1) / num_pts_tmp count += 1 print 'number of skipped images is %d' % count_skip_num_images assert count + count_skip_num_images == num_images, 'all cells in the array must be filled %d vs %d' % (count + count_skip_num_images, num_images) # print normed_mean_error_total # time.sleep(1000) # save results to dictionary normed_mean_error_dict[method_name] = normed_mean_error_total[:count] normed_mean_error_pts_specific_dict[method_name] = normed_mean_error_pts_specific[:count, :] normed_mean_error_pts_specific_valid_dict[method_name] = normed_mean_error_pts_specific_valid[:count, :] pts_error_vec_dict[method_name] = np.transpose(pts_error_vec[:count, :]) # 2 x num_images pts_error_vec_pts_specific_dict[method_name] = pts_error_vec_pts_specific[:count, :, :] mse_error_dict_dict[method_name] = mse_error_dict # calculate mean value if mse: mse_value = dict() # dictionary to record all average MSE for different methods mse_dict = dict() # dictionary to record all point-wise MSE for different keypoints for method_name, error_array in normed_mean_error_dict.items(): mse_value[method_name] = np.mean(error_array) else: mse_value = None # save mse error list to file for each method error_list_savedir = os.path.join(save_path, 'error_list') mkdir_if_missing(error_list_savedir) for method_name, mse_error_dict in mse_error_dict_dict.items(): mse_error_list_path = os.path.join(error_list_savedir, 'error_%s.txt' % method_name) mse_error_list = open(mse_error_list_path, 'w') sorted_tuple_list = sorted(mse_error_dict.items(), key=operator.itemgetter(1), reverse=True) for tuple_index in range(len(sorted_tuple_list)): image_path_tmp = sorted_tuple_list[tuple_index][0] mse_error_tmp = sorted_tuple_list[tuple_index][1] mse_error_list.write('{:<200} {}\n'.format(image_path_tmp, '%.2f' % mse_error_tmp)) mse_error_list.close() print '\nsave mse error list for %s to %s' % (method_name, mse_error_list_path) # visualize the ced (cumulative error distribution curve) print('visualizing pck curve....\n') pck_savedir = os.path.join(save_path, 'pck') mkdir_if_missing(pck_savedir) pck_savepath = os.path.join(pck_savedir, 'pck_curve_overall.png') table_savedir = os.path.join(save_path, 'metrics') mkdir_if_missing(table_savedir) table_savepath = os.path.join(table_savedir, 'detailed_metrics_overall.txt') _, metrics_all = visualize_ced(normed_mean_error_dict, error_threshold=error_threshold, normalized=normalization_ced, truncated_list=truncated_list, title='2D PCK curve (all %d points)' % num_pts, display_list=display_list, debug=debug, vis=vis, pck_savepath=pck_savepath, table_savepath=table_savepath) metrics_title = ['Method Name / Point Index'] ptswise_mse_table = [[normed_mean_error_pts_specific_dict.keys()[index_tmp]] for index_tmp in xrange(num_methods)] for pts_index in xrange(num_pts): metrics_title.append(str(pts_index + 1)) normed_mean_error_dict_tmp = dict() for method_name, error_array in normed_mean_error_pts_specific_dict.items(): normed_mean_error_pts_specific_valid_temp = normed_mean_error_pts_specific_valid_dict[method_name] # Some points at certain images might not be annotated. When calculating MSE for these specific point, we remove those images to avoid "false" mean average error valid_array_per_pts_per_method = np.where(normed_mean_error_pts_specific_valid_temp[:, pts_index] == True)[0].tolist() error_array_per_pts = error_array[:, pts_index] error_array_per_pts = error_array_per_pts[valid_array_per_pts_per_method] num_image_tmp = len(valid_array_per_pts_per_method) # print(num_image_tmp) if num_image_tmp == 0: continue # aaa normed_mean_error_dict_tmp[method_name] = np.reshape(error_array_per_pts, (num_image_tmp, )) pck_savepath = os.path.join(pck_savedir, 'pck_curve_pts_%d.png' % (pts_index+1)) table_savepath = os.path.join(table_savedir, 'detailed_metrics_pts_%d.txt' % (pts_index+1)) if len(normed_mean_error_dict_tmp) == 0: continue metrics_dict, _ = visualize_ced(normed_mean_error_dict_tmp, error_threshold=error_threshold, normalized=normalization_ced, truncated_list=truncated_list, display2terminal=False, title='2D PCK curve for point %d' % (pts_index+1), display_list=display_list, debug=debug, vis=vis, pck_savepath=pck_savepath, table_savepath=table_savepath) for method_index in range(num_methods): method_name = normed_mean_error_pts_specific_dict.keys()[method_index] ptswise_mse_table[method_index].append('%.1f' % metrics_dict[method_name]['MSE']) # reorder the table order_index_list = [display_list.index(method_name_tmp) for method_name_tmp in normed_mean_error_pts_specific_dict.keys()] order_index_list = [0] + [order_index_tmp + 1 for order_index_tmp in order_index_list] # print table to terminal ptswise_mse_table = list_reorder([metrics_title] + ptswise_mse_table, order_index_list, debug=debug) table = AsciiTable(ptswise_mse_table) print '\nprint point-wise average MSE' print table.table # save table to file ptswise_savepath = os.path.join(table_savedir, 'pointwise_average_MSE.txt') table_file = open(ptswise_savepath, 'w') table_file.write(table.table) table_file.close() print '\nsave point-wise average MSE to %s' % ptswise_savepath # visualize the error vector map # print('visualizing error vector distribution map....\n') # error_vec_save_dir = os.path.join(save_path, 'error_vec') # mkdir_if_missing(error_vec_save_dir) # savepath_tmp = os.path.join(error_vec_save_dir, 'error_vector_distribution_all.png') # visualize_pts(pts_error_vec_dict, title='Point Error Vector Distribution (all %d points)' % num_pts, mse=mse, mse_value=mse_value, display_range=display_range, display_list=display_list, xlim=xlim, ylim=ylim, covariance=covariance, debug=debug, vis=vis, save_path=savepath_tmp) # for pts_index in xrange(num_pts): # pts_error_vec_pts_specific_dict_tmp = dict() # for method_name, error_vec_dict in pts_error_vec_pts_specific_dict.items(): # pts_error_vec_pts_specific_valid = normed_mean_error_pts_specific_valid_dict[method_name] # get valid flag # valid_image_index_per_pts = np.where(pts_error_vec_pts_specific_valid[:, pts_index] == True)[0].tolist() # get images where the points with current index are annotated # print(len(valid_image_index_per_pts)) # pts_error_vec_pts_specific_dict_tmp[method_name] = np.transpose(error_vec_dict[valid_image_index_per_pts, :, pts_index]) # 2 x num_images # savepath_tmp = os.path.join(error_vec_save_dir, 'error_vector_distribution_pts_%d.png' % (pts_index+1)) # if mse: # mse_dict_tmp = visualize_pts(pts_error_vec_pts_specific_dict_tmp, title='Point Error Vector Distribution for Point %d' % (pts_index+1), mse=mse, display_range=display_range, display_list=display_list, xlim=xlim, ylim=ylim, covariance=covariance, debug=debug, vis=vis, save_path=savepath_tmp) # mse_best = min(mse_dict_tmp.values()) # mse_single = dict() # mse_single['mse'] = mse_best # mse_single['num_images'] = len(valid_image_index_per_pts) # assume number of valid images is equal for all methods # mse_dict[pts_index] = mse_single # else: # visualize_pts(pts_error_vec_pts_specific_dict_tmp, title='Point Error Vector Distribution for Point %d' % (pts_index+1), mse=mse, display_range=display_range, display_list=display_list, xlim=xlim, ylim=ylim, covariance=covariance, debug=debug, vis=vis, save_path=savepath_tmp) # save mse to json file for further use # if mse: # json_path = os.path.join(save_path, 'mse_pts.json') # # if existing, compare and select the best # if is_path_exists(json_path): # with open(json_path, 'r') as file: # mse_dict_old = json.load(file) # file.close() # for pts_index, mse_single in mse_dict_old.items(): # mse_dict_new = mse_dict[int(pts_index)] # mse_new = mse_dict_new['mse'] # if mse_new < mse_single['mse']: # mse_single['mse'] = mse_new # mse_dict_old[pts_index] = mse_single # with open(json_path, 'w') as file: # print('overwrite old mse to {}'.format(json_path)) # json.dump(mse_dict_old, file) # file.close() # else: # with open(json_path, 'w') as file: # print('save mse for all keypoings to {}'.format(json_path)) # json.dump(mse_dict, file) # file.close() print('\ndone!!!!!\n') return metrics_all, ptswise_mse_table
def compute_metrics(gt_json, pred_json, split='test', total_sample=20): results_dir = fileparts(pred_json)[0] log_file = os.path.join(results_dir, 'eval_traj_log.txt') log_file = open(log_file, 'w') # data loading print_log('loading GT from %s' % gt_json, log_file) with open(gt_json, 'r') as file: gt_data = json.load(file) print_log('loading results from %s' % pred_json, log_file) with open(pred_json, 'r') as file: pred_data = json.load(file) # 4 metrics x 3 settings x 4 classes x N windows metrics = ['ADE', 'FDE', 'APD', 'FPD', 'MsR'] # set up metrics stats_meter = {} for pred_len in [10, 20, 50]: # 1s, 2s, 5s prediction settings skip = len2skip[pred_len] for obj_class in ['Car', 'Ped', 'Cyc', 'Mot']: for metric_tmp in metrics: stats_meter['%s_%d_%s' % (metric_tmp, pred_len, obj_class)] = AverageMeter() gt_obj = gt_data[obj_class] if str(pred_len) not in pred_data.keys(): continue pred_data_len = pred_data[str( pred_len)] # get prediction at a specific pred len if obj_class not in pred_data_len.keys(): continue pred_data_obj = pred_data_len[ obj_class] # get prediction for a specific class # loop through based on GT data so that missing prediction can be handled num_total_obj_valid, num_miss = 0, 0 for seqname, gt_seq in gt_obj.items(): gt_seq = np.array(gt_seq) # N x 4 # frame-centric evaluation, go through every possible window of frames, e.g., frame 1-20, 2-21 for 10->10 frames = np.unique(gt_seq[:, 0]).tolist() min_frame, max_frame = frames[0], frames[-1] num_windows = int(max_frame - min_frame + 1 - pred_len * 2 + skip) # this additional + 1 is correct for window_index in range(num_windows): start_frame = int(window_index + min_frame + pred_len) end_frame = int(start_frame + pred_len) # check which window to be dropped check_pass = check_eval_windows(start_frame, pred_len, split) if not check_pass: continue ####### filter frame windows, reserver any window with past data >= 1 frame, future data >= 1 frame # filter out frame window where there is no GT inside gt_window = [] for frame in range(start_frame, end_frame, skip): gt_window.append(gt_seq[frame == gt_seq[:, 0], :]) gt_window = np.concatenate( gt_window, axis=0) # N x 4, within a particular time window if gt_window.shape[0] == 0: continue ID_list = np.unique(gt_window[:, 1]).tolist() # get GT IDs # filter out frame window where there is no past observations at all past_start = start_frame - pred_len gt_window_past = [] for frame in range(past_start, start_frame, skip): gt_window_past.append(gt_seq[frame == gt_seq[:, 0], :]) gt_window_past = np.concatenate( gt_window_past, axis=0) # N x 4, within a particular time window if gt_window_past.shape[0] == 0: continue # filter out ID in this frame window when this ID has no past observations for ID_tmp in ID_list: gt_past = copy.copy( gt_window_past[gt_window_past[:, 1] == float(ID_tmp), :] ) # (<=pred_len/skip) x 2, might have missing data num_past_frames = gt_past.shape[0] if num_past_frames == 0: ID_list, _ = remove_unique_item_from_list( ID_list, ID_tmp) if len(ID_list) == 0: continue num_total_obj_valid += len(ID_list) # check prediction if have data in this seq and frame, otherwise all objects are missed try: pred_window = pred_data_obj[seqname][str( start_frame)] # {sample: {ID: {state, prob}}} except: try: pred_window = pred_data_obj[seqname][ start_frame] # {sample: {ID: {state, prob}}} except: print('No prediction data found for len of %d, %s, seq %s, frame %s' % \ (pred_len, obj_class, seqname, start_frame)) num_miss += len(ID_list) continue ######## get predictions within this window sample_count = 0 ade_list, fde_list = [], [] pred_all_sample, gt_all_sample = [], [] # sample must be in the outer loop to compute scene-specific min of ADE for sample_index, pred_tmp in pred_window.items(): best_k_pred, best_k_gt = [], [] gt_mask = np.zeros( (len(ID_list), int(pred_len / skip))) # initialize mask on missing frames keep_ID_index = [] for ID_count in range(len(ID_list)): ID_tmp = ID_list[ID_count] # check if prediction has data for this ID, otherwise this object is missed try: pred_ID = pred_tmp[str(int(ID_tmp))] except: try: pred_ID = pred_tmp[int(ID_tmp)] except: if int(sample_index) == 0: num_miss += 1 print('\nID %d missied in prediction data for len of %d, %s, seq %s, frame %s\n' % \ (ID_tmp, pred_len, obj_class, seqname, start_frame)) continue # get prediction states state, prob = pred_ID['state'], pred_ID['prob'] state = np.array(state) # pred_len/skip x 2 assert state.shape[0] == pred_len / skip, 'error' # get GT states gt_ID = copy.copy( gt_window[gt_window[:, 1] == float(ID_tmp), :] ) # (<=pred_len/skip) x 2, might have missing data assert gt_ID.shape[ 0] > 0, 'error' # this is guaranteed, as this ID exists so there is at least one frame of data for this ID frame_exist = gt_ID[:, 0].tolist( ) # get existing frames before completion if gt_ID.shape[0] < pred_len / skip: gt_ID = complete_data_window( gt_ID, start_frame, end_frame, skip) # pred_len/skip x 2 # get GT mask frame_exist_index = np.array([ frame_tmp - start_frame for frame_tmp in frame_exist ]) frame_exist_index = (frame_exist_index / skip).astype('uint8') gt_mask[ID_count, frame_exist_index] = 1 # store both prediction and GT keep_ID_index.append(ID_count) best_k_pred.append(state) best_k_gt.append( gt_ID[:, 2:] ) # take only the state (last two columns) # compute ADE/FDE in this sample gt_mask = gt_mask[keep_ID_index, :] best_k_pred = np.stack( best_k_pred, axis=0) # obj x pred_len/skip x 2 best_k_gt = np.stack(best_k_gt, axis=0) # obj x pred_len/skip x 2 ade_tmp = compute_ADE(best_k_pred, best_k_gt, mask=gt_mask) fde_tmp = compute_FDE(best_k_pred, best_k_gt, mask=gt_mask) ade_list.append(ade_tmp) fde_list.append(fde_tmp) # aggregate all samples pred_all_sample.append(best_k_pred) gt_all_sample.append(best_k_gt) # evaluation allows up to the first 20 samples sample_count += 1 if sample_count > total_sample: break # check if every sample has the same shape, i.e., has the same number of predictions of IDs try: ade_array = np.stack(ade_list, axis=1) # obj x samples fde_array = np.stack(fde_list, axis=1) # obj x samples pred_all_sample = np.stack( pred_all_sample, axis=0) # samples x obj x frames x 2 gt_all_sample = np.stack(gt_all_sample, axis=0) assert pred_all_sample.shape == gt_all_sample.shape, 'error' except ValueError: assert False, 'each sample has predictions with different set of IDs, please make sure every sample has the same shape of prediction (same set of IDs)' # compute best of K ADE/FDE ade_best, fde_best = best_of_K(ade_array, fde_array) num_obj = ade_array.shape[ 0] # might less than len(ID_list) as some IDs are not contained in predictions stats_meter['ADE_%s_%s' % (pred_len, obj_class)].update( ade_best, n=num_obj) stats_meter['FDE_%s_%s' % (pred_len, obj_class)].update( fde_best, n=num_obj) # compute APD/FPD for diversity apd = compute_APD(pred_all_sample) fpd = compute_FPD(pred_all_sample) stats_meter['APD_%s_%s' % (pred_len, obj_class)].update( apd, n=num_obj) stats_meter['FPD_%s_%s' % (pred_len, obj_class)].update( fpd, n=num_obj) # only copying the ones under the current class when logging stats_meter_show = {} for metric_tmp, values in stats_meter.items(): if (str(pred_len) in metric_tmp) and obj_class in metric_tmp: stats_meter_show[metric_tmp] = values # logging stats_str = ' '.join([ f'{x[:3]}: {y.val:6.3f} ({y.avg:6.3f})' for x, y in stats_meter_show.items() ]) print_str = 'Len: %d, %s, %s, %03d-%03d, %s\r' % \ (pred_len, obj_class, seqname, start_frame, end_frame-1, stats_str) print_log(print_str, log_file, display=False) sys.stdout.write(print_str) sys.stdout.flush() # compute missing rate metric missrate = num_miss * 1.0 / num_total_obj_valid stats_meter['MsR_%s_%s' % (pred_len, obj_class)].update( missrate, n=num_total_obj_valid) assert missrate == stats_meter['MsR_%s_%s' % (pred_len, obj_class)].avg, 'error' if num_miss > 0: print('\n\nMissing IDs: %d, total IDs: %d, miss rate: %.3f for len of %d, %s\n' % \ (num_miss, num_total_obj_valid, missrate, pred_len, obj_class)) ################ logging print_log('', log_file) metric_all = dict() for pred_len in [10, 20, 50]: # compute average ADE/FDE/APD/FPD/MsR for metric in ['ADE', 'FDE', 'APD', 'FPD', 'MsR']: metric_all['%s_%d' % (metric, pred_len)] = list() for obj_class in ['Car', 'Ped', 'Cyc', 'Mot']: key_tmp = '%s_%d_%s' % (metric, pred_len, obj_class) num_value = stats_meter[key_tmp].count if num_value == 0: tmp_value = np.nan else: tmp_value = stats_meter[key_tmp].avg metric_all['%s_%d' % (metric, pred_len)].append(tmp_value) metric_all['%s_%d' % (metric, pred_len)] = np.array( metric_all['%s_%d' % (metric, pred_len)]).mean() # logging header print_log('', log_file) print_log('-' * 30 + ' STATS for length of %d ' % pred_len + '-' * 30, log_file) print_log(' ' * 5, log_file, same_line=True) print_log('ADE_%d'%pred_len+' '*2+'FDE_%d'%pred_len+' '*2+'APD_%d'%pred_len+' '*2+'FPD_%d'%pred_len+' '*2+'MsR_%d'%pred_len+' '*2, \ log_file, same_line=True) print_log('', log_file) # logging table: row class, column metrics cur_class = 'Car' cur_index = 0 for name, meter in stats_meter.items(): num_value = meter.count if num_value == 0: final_value = np.nan else: final_value = meter.avg if str(pred_len) not in name: continue # check to see if switch from one class to the other if cur_class not in name: print_log('', log_file) cur_class = name[-3:] cur_index = 0 # print all errors for this class if cur_index == 0: print_log('%s: ' % name[-3:], log_file, same_line=True) print_log('%6.3f ' % final_value, log_file, same_line=True) cur_index += 1 # print average value print_log('', log_file) print_log('Ave: ', log_file, same_line=True) for metric in ['ADE', 'FDE', 'APD', 'FPD', 'MsR']: ave_value = metric_all['%s_%d' % (metric, pred_len)] print_log('%6.3f ' % ave_value, log_file, same_line=True) # print end of the logging print_log('', log_file) print_log('-' * 84, log_file) log_file.close() # filter out metrics other than len of 20 output = dict() for key, value in metric_all.items(): if '20' in key: # only add length of 20 to output metric for EvalAI leaderboard output[key] = value pprint.pprint(output) # print output metrics return output