def copy_data(): info = ulti.load_json() dir_images = os.path.join(info['dataset_dir'], '..', 'Raw') dir_output = os.path.join(info['dataset_dir'], 'Images') videos = os.listdir(dir_images) for video in sorted(videos): if 'frankfurt' != video: pass for id in range(1000): prex = (video + '_%06d') % id files = glob(os.path.join(dir_images, video, prex) + '_*') if len(files) > 0: files = sorted(files) if not os.path.exists(os.path.join(dir_output, prex)): os.makedirs(os.path.join(dir_output, prex)) for file in files: img = cv2.imread(os.path.join(dir_images, video, file)) print( os.path.join( dir_output, prex, os.path.splitext(os.path.basename(file))[0] + '.jpg')) cv2.imwrite( os.path.join( dir_output, prex, os.path.splitext(os.path.basename(file))[0] + '.jpg'), img) else: for id in range(1000): prex = video + '_%06d' % id files = glob(os.path.join(dir_images, video, prex) + '_*') files = [os.path.splitext(file)[0] for file in files] files = list(dict.fromkeys(files)) if len(files) > 0: files = sorted(files) count = 0 for file in files: # print('{} {}'.format(count//30, count%30)) new_prex = prex + '_%06d' % ((count) // 30) if (count % 30) == 0: if not os.path.exists( os.path.join(dir_output, new_prex)): os.makedirs(os.path.join(dir_output, new_prex)) if os.path.isfile( os.path.join(dir_images, video, file + '.jpg')): img = cv2.imread( os.path.join(dir_images, video, file + '.jpg')) else: img = cv2.imread( os.path.join(dir_images, video, file + '.png')) cv2.imwrite( os.path.join( dir_output, new_prex, os.path.splitext(os.path.basename(file))[0] + '.jpg'), img) files_tmp = os.listdir( os.path.join(dir_output, new_prex)) count += 1
def create_test(): info = ulti.load_json() dir_input = info['dataset_dir'] dir_output = os.path.join(info['dataset_dir'], 'RCNN_data') video = info['annotated_video'] exclude_true_gt_frames = False path = os.path.join(dir_input, 'Info', video + '.json') dataset = ulti.load_json(dir_input + '/Info/' + video + '.json') # pprint(dataset) videos = dataset['videos'] images = [] path = dir_input + '/Info/' + video + '.json' if os.path.isfile(path): data = ulti.load_json(path) if exclude_true_gt_frames: for image in data['images']: if not image['has_gt']: images.append(image) else: images.extend(data['images']) tq = tqdm.tqdm(total=len(images)) for image in images: tq.update(1) image['file_name'] = image['file_name'].replace('Images\\\\', '') image['file_name'] = image['file_name'].replace('Images/', '') image['file_name'] = image['file_name'].replace('Images\\', '') categories = ulti.load_json(dir_input + '/Categories/Road_Objects.json') categories = categories['category'] dataset = { 'categories': categories, 'annotations': [], 'videos': [], 'images': [] } dataset['videos'] = videos dataset['images'] = images print('test: ', len(images)) # pprint(dataset) ulti.make_dir(dir_output) outfile = ulti.write_json(dataset, file=os.path.join(dir_output, 'test.json'))
def filter_data(videonames=[]): info = ulti.load_json() if not videonames: videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) tq = tqdm.tqdm(total=len(videonames)) frequency_threshold = 4 for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) dir_tubelet = os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Tubelet', videoname + '.json') ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Videos', videoname)) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Tubelet')) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Visualization', videoname)) with open(dir_tubelet) as f: tubelet = json.load(f) tubelet_by_frame = tubelet['tubelet_by_frame'] tubelet_by_id = tubelet['tubelet_by_id'] for id in list(tubelet_by_id.keys()): tubelet = tubelet_by_id[id] if len(tubelet.keys()) < frequency_threshold: for frame in tubelet.keys(): del tubelet_by_frame[frame][id] del tubelet_by_id[id] continue for frame in list(tubelet.keys()): img = cv2.imread(os.path.join(info['dataset_dir'], 'Images', videoname, frame + '.jpg')) height, width, channels = img.shape bbox = tubelet[frame] bbox = check_condition(bbox, height, width) if bbox is None: del tubelet_by_id[id][frame] del tubelet_by_frame[frame][id] if len(tubelet_by_frame[frame].keys()) == 0: del tubelet_by_frame[frame] if len(tubelet_by_id[id].keys()) == 0: del tubelet_by_id[id] ulti.write_json({'tubelet_by_frame': tubelet_by_frame, 'tubelet_by_id': tubelet_by_id}, os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Tubelet', videoname + '.json')) for frame in tubelet_by_frame.keys(): tubelet = tubelet_by_frame[frame] ulti.write_json(tubelet, os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Videos', videoname, frame + '.json'))
def create_attributes(): info = ulti.load_json() output_dir = os.path.join(info['dataset_dir'], 'Categories') ulti.make_dir(output_dir) info = {} info['id'] = 1 info['name'] = 'Road_Objects' category = [ { "id": 1, "name": "pedestrian", "type": "thing", "supercategory": "person" }, { "id": 2, "name": "rider", "type": "thing", "supercategory": "person" }, { "id": 3, "name": "car", "type": "thing", "supercategory": "vehicle" }, { "id": 4, "name": "truck", "type": "thing", "supercategory": "vehicle" }, { "id": 5, "name": "bus", "type": "thing", "supercategory": "vehicle" }, { "id": 6, "name": "motorcycle", "type": "thing", "supercategory": "vehicle" }, { "id": 7, "name": "bicycle", "type": "thing", "supercategory": "vehicle" }, ] info['category'] = category outfile = ulti.write_json(info, file=os.path.join(output_dir, info['name'] + '.json'))
def main(model_path=''): ulti.copy_model(model_path=model_path) info = ulti.load_json() training_dir = info['training_dir'] dataset_name = info['annotated_video'] dir_input = os.path.join(training_dir, dataset_name) if os.path.exists(dir_input): shutil.rmtree(dir_input, ignore_errors=True) src = os.path.join( info['dataset_dir'], '../Initial_model/e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml') dst = os.path.join(info['training_dir'], 'e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml') shutil.copyfile(src, dst)
def smoothen_label(videonames=[]): info = ulti.load_json() if not videonames: videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) tq = tqdm.tqdm(total=len(videonames)) for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) dir_tubelet = os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Tubelet', videoname + '.json') ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Videos', videoname)) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Tubelet')) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Visualization', videoname)) with open(dir_tubelet) as f: tubelet = json.load(f) tubelet_by_frame = tubelet['tubelet_by_frame'] tubelet_by_id = tubelet['tubelet_by_id'] for id in tubelet_by_id.keys(): tubelet = tubelet_by_id[id] category_ids = [] category_scores = [] for frame in tubelet.keys(): bbox = tubelet[frame] category_ids.append(bbox['category_id']) category_scores.append(bbox['score']) # if len(np.unique(np.array(category_ids))) > 1: element, score = most_frequent(category_ids, category_scores) # print(frame, id, element, ':', category_ids) for frame in tubelet.keys(): if score: tubelet_by_id[id][frame]['score'] = score tubelet_by_frame[frame][id]['score'] = score tubelet_by_id[id][frame]['category_id'] = element tubelet_by_frame[frame][id]['category_id'] = element ulti.write_json({'tubelet_by_frame': tubelet_by_frame, 'tubelet_by_id': tubelet_by_id}, os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Tubelet', videoname + '.json')) for frame in tubelet_by_frame.keys(): tubelet = tubelet_by_frame[frame] ulti.write_json(tubelet, os.path.join(info['dataset_dir'], info['experiment'], 'Smooth_label', 'Videos', videoname, frame + '.json'))
def convert_args(): info = ulti.load_json() parser = argparse.ArgumentParser( description="PyTorch Object Detection Training") parser.add_argument("--local_rank", type=int, default=0) parser.add_argument( "--config-file", default=os.path.join(info['training_dir'], 'e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml'), metavar="FILE", help="path to config file", type=str, ) parser.add_argument( "--sub-dataset", dest="sub_dataset", help="Video Name", default=info['annotated_video'], # default=None, type=str, ) parser.add_argument( "--epochs", dest="epochs", help="Number of Epochs", default=25, type=int, ) parser.add_argument( "--scale", dest="scale", help="Scale of Epochs", default=0.5, type=int, ) parser.add_argument( "--train", dest="train", help="Train model", default=False, type=bool, ) parser.add_argument( "--test", dest="test", help="Test the final model", default=False, type=bool, ) parser.add_argument( "--visualize-loss", dest="visualize_loss", help="Draw loss function", default="", # "visdom", "tensorboardx", "tensorboard_logger" type=str, ) parser.add_argument("--visualize-results", dest="visualize_results", help="Visualize results", default=False, type=bool) parser.add_argument("--only-visualization", dest="only_visualization", help="Only visualize results, not evaluate results", default=True, type=bool) parser.add_argument("--category", dest="category", help="dataset having categories", default="ro", type=str) parser.add_argument( "--visualization-title", dest="vis_title", help="Visualization Title", default=None, type=str, ) parser.add_argument( "--ckpt", dest="ckpt", help= "The path to the checkpoint for test, default is the latest checkpoint.", default=None, ) parser.add_argument( "--copy-weight-from-head-box", dest="copy_weight_from_head_box", help= "Copy weight from ROI head box to other branches if they are initialized", default=False, type=bool) args, _ = parser.parse_known_args() num_gpus = int( os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1 args.distributed = num_gpus > 1 if args.distributed: torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend="nccl", init_method="env://") synchronize() return args
def main(args): seed_torch() info = ulti.load_json() num_gpus = get_num_gpus() args.config_file = os.path.join( info['training_dir'], 'e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml') cfg.merge_from_file(args.config_file) cfg.defrost() cfg.OUTPUT_DIR = os.path.join(info['training_dir'], args.sub_dataset) cfg.MODEL.WEIGHT = os.path.join(info['dataset_dir'], info['experiment'], 'Detector', 'Iter{}.pth'.format(info['iter'])) cfg.SOLVER.IMS_PER_BATCH = num_gpus * 4 cfg.TEST.IMS_PER_BATCH = num_gpus * 16 cfg.SOLVER.BASE_LR = 0.002 cfg.freeze() mkdir(cfg.OUTPUT_DIR) if args.sub_dataset is None: args.sub_dataset = "" if args.vis_title is None: args.vis_title = os.path.basename(cfg.OUTPUT_DIR) logger = setup_logger("maskrcnn_benchmark", cfg.OUTPUT_DIR, get_rank()) logger.info("Using {} GPUs".format(num_gpus)) logger.info(args) logger.info("Collecting env info (might take some time)") # logger.info("\n" + collect_env_info()) DatasetCatalog = None train_dataset = cfg.DATASETS.TRAIN[0] test_dataset = cfg.DATASETS.TEST[0] paths_catalog = import_file("maskrcnn_benchmark.config.paths_catalog", cfg.PATHS_CATALOG, True) if args.sub_dataset != "": DatasetCatalog = paths_catalog.DatasetCatalog DatasetCatalog.DATASETS[train_dataset]['img_dir'] = os.path.join( info['dataset_dir'], 'Images') DatasetCatalog.DATASETS[train_dataset]['ann_file'] = os.path.join( info['dataset_dir'], 'RCNN_data', 'train.json') DatasetCatalog.DATASETS[test_dataset]['img_dir'] = os.path.join( info['dataset_dir'], 'Images') DatasetCatalog.DATASETS[test_dataset]['ann_file'] = os.path.join( info['dataset_dir'], 'RCNN_data', 'test.json') data = json.load( open(DatasetCatalog.DATASETS[train_dataset]['ann_file'])) else: data = json.load( open(paths_catalog.DatasetCatalog.DATASETS[train_dataset] ['ann_file'])) iters_per_epoch = len(data['images']) iters_per_epoch = math.ceil(iters_per_epoch / cfg.SOLVER.IMS_PER_BATCH) args.iters_per_epoch = iters_per_epoch cfg.defrost() cfg.SOLVER.MAX_ITER = round(args.epochs * args.scale * iters_per_epoch) cfg.SOLVER.STEPS = (round(8 * args.scale * iters_per_epoch), round(11 * args.scale * iters_per_epoch), round(16 * args.scale * iters_per_epoch)) cfg.freeze() # logger.info("Loaded configuration file {}".format(args.config_file)) with open(args.config_file, "r") as cf: config_str = "\n" + cf.read() # logger.info(config_str) # logger.info("Running with config:\n{}".format(cfg)) # logger.info(DatasetCatalog) output_config_path = os.path.join(cfg.OUTPUT_DIR, 'config.yml') logger.info("Saving config into: {}".format(output_config_path)) # save overloaded model config in the output directory save_config(cfg, output_config_path) if args.train: args.skip_train = False logger.info(args) model = network.train(cfg, args, DatasetCatalog) if args.test: network.test(cfg, args, model=None, DatasetCatalog=DatasetCatalog)
def create_train(): info = ulti.load_json() dir_input = info['dataset_dir'] dir_output = os.path.join(info['dataset_dir'], 'RCNN_data') video = info['annotated_video'] only_use_true_gt = False path = os.path.join(dir_input, 'Info', video + '.json') dataset = ulti.load_json(path) images = dataset['images'] videos = dataset['videos'] categories = ulti.load_json(dir_input + '/Categories/Road_Objects.json') categories = categories['category'] dataset = { 'categories': categories, 'annotations': [], 'videos': [], 'images': [] } video_names = [] video_ids = [] if video == info['dataset_name']: for vid in videos: dataset['videos'].append(vid) video_names.append(vid['name']) video_ids.append(vid['id']) else: for vid in videos: if vid['name'] == video: dataset['videos'].append(vid) video_names.append(vid['name']) video_ids.append(vid['id']) for image in images: image['file_name'] = image['file_name'].replace('Images\\\\', '') image['file_name'] = image['file_name'].replace('Images/', '') image['file_name'] = image['file_name'].replace('Images\\', '') ann_files = [] list_images = [] annotations = [] ins_id = 0 tq = tqdm.tqdm(total=len(video_names)) for id, video in zip(video_ids, video_names): tq.update(1) for (dirpath, dirnames, filenames) in os.walk( os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', video)): ann_temp = [] img_temp = [] for file in filenames: if file.endswith('.json'): data = ulti.load_json(os.path.join(dirpath, file)) if len(data) > 0: ann_files.append(file) ann_temp.append(file) img_temp = sorted(img_temp) ann_temp = sorted(ann_temp) for image in images: if image['video_id'] == id: file_name = os.path.splitext( os.path.basename(image['file_name']))[0] + '.json' if file_name in ann_temp: if (image['has_gt'] and only_use_true_gt) or not only_use_true_gt: # print(file_name) list_images.append(image) img_temp.append(image) json_file = os.path.splitext( os.path.basename( image['file_name']))[0] + '.json' if json_file in ann_temp: data = ulti.load_json( os.path.join(dirpath, json_file)) for ann in data: ann['bbox'] = [ ann['bbox']['x'], ann['bbox']['y'], ann['bbox']['w'], ann['bbox']['h'] ] ann['id'] = ins_id ann['image_id'] = image['id'] ins_id += 1 annotations.extend(data) tq.set_description('Video {}'.format( os.path.join(video, file_name))) dataset['annotations'] = annotations dataset['videos'] = videos dataset['images'] = list_images ulti.make_dir(dir_output) outfile = ulti.write_json(dataset, file=os.path.join(dir_output, 'train.json'))
def create_dataset_info(img_id_start=0, video_id_start=0, ann_id_start=0): info = ulti.load_json() dataset_dir = os.path.join(info['dataset_dir']) dataset_name = info['dataset_name'] img_id = img_id_start video_id = video_id_start ann_id = ann_id_start dataset = {} dataset['id'] = 1 info = {} info['name'] = dataset_name info['root_dir'] = dataset_dir + '/' info['type'] = 'video' # 'video' or 'image' info['ann_dir'] = '' info['extension'] = 'jpg' dataset['info'] = info videos = [] data_dir = os.path.join(info['root_dir'], 'Images') folders = [ f for f in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, f)) ] for folder in sorted(folders): video = {} video['id'] = video_id video['name'] = folder data_dir = os.path.join(info['root_dir'], 'Images', video['name']) files = [ f for f in os.listdir(data_dir) if os.path.isfile(os.path.join(data_dir, f)) and f.endswith(info['extension']) ] video['n_frames'] = len(files) videos.append(video) video_id += 1 dataset['videos'] = videos images = [] tq = tqdm.tqdm(total=len(videos)) for video in videos: tq.update(1) data_dir = os.path.join(info['root_dir'], 'Images', video['name']) files = [ f for f in os.listdir(data_dir) if os.path.isfile(os.path.join(data_dir, f)) and f.endswith(info['extension']) ] sub_images = [] sub_dataset = copy.deepcopy(dataset) for f in sorted(files): image = {} image['id'] = img_id image['has_gt'] = False image['video_id'] = video['id'] image['file_name'] = os.path.join('Images', video['name'], f) image['seg_file_name'] = '' img = Image.open(os.path.join(info['root_dir'], image['file_name'])).convert("RGB") img = np.array(img)[:, :, [2, 1, 0]] img = img.copy() image['width'] = img.shape[1] image['height'] = img.shape[0] img_id += 1 images.append(image) sub_images.append(image) tq.set_description('Video {}'.format(image['file_name'])) sub_dataset['images'] = sub_images dir_output = dataset_dir + '/Info' ulti.make_dir(dir_output) outfile = ulti.write_json(sub_dataset, file=(dir_output + '/' + video['name'] + '.json')) dataset['images'] = images dir_output = dataset_dir + '/Info/' ulti.make_dir(dir_output) outfile = ulti.write_json(dataset, file=(dir_output + '/' + info['name'] + '.json'))
def generate_ann(threshold=None, trackable_threshold=0): if threshold is None: threshold = [0.85, 0.550, 0.950, 0.950, 0.800, 0.800, 0.700] info = ulti.load_json() video = info['annotated_video'] dataset_dir = info['dataset_dir'] output_dir = os.path.join(dataset_dir, info['experiment'], 'Detection') ulti.make_dir(output_dir) print(output_dir) input_path = os.path.join(dataset_dir, info['experiment'], 'Raw_Detection', 'bbox.json') prediction = ulti.load_json(input_path) category = ulti.load_json( os.path.join(dataset_dir, 'Categories', 'Road_Objects.json')) dataset = ulti.load_json(os.path.join(dataset_dir, 'Info', video + '.json')) list_img_id_dataset = [] list_img_id_prediction = [] ann_id = 0 for ann in prediction: list_img_id_prediction.append(ann['image_id']) tq = tqdm.tqdm(total=len(dataset['images'])) for img in dataset['images']: list_img_id_dataset.append(img['id']) tq.set_description('Video Frame {}'.format(img['id'])) tq.update(1) index = [ i for i, x in enumerate(list_img_id_prediction) if x == img['id'] and prediction[i]['score'] >= threshold[ prediction[i]['category_id'] - 1] ] if len(index) > 0: # print(img['id'], index) annotations = [] for ind in index: ann = {} ann['id'] = ann_id ann['image_id'] = prediction[ind]['image_id'] ann['track_id'] = -1 ann['category_id'] = prediction[ind]['category_id'] ann['score'] = prediction[ind]['score'] ann['second_category_id'] = None ann['second_score'] = 0 ann['third_category_id'] = None ann['third_score'] = 0 ann['iscrowd'] = 0 # always 0 ann['bbox'] = { 'x': int(prediction[ind]['bbox'][0]), 'y': int(prediction[ind]['bbox'][1]), 'w': int(prediction[ind]['bbox'][2]), 'h': int(prediction[ind]['bbox'][3]) } if ann['bbox']['w'] * ann['bbox']['h'] >= trackable_threshold: ann['trackable'] = True else: ann['trackable'] = False ann['segmentation'] = [] ann['area'] = 0 ann['score'] = prediction[ind]['score'] annotations.append(ann) ann_id += 1 if len(annotations) > 0: fullpath = os.path.join( dataset_dir, 'Annotations', category['name'], os.path.basename(os.path.dirname(img['file_name']))) ulti.make_dir(fullpath) fullpath = os.path.join( output_dir, 'Json', os.path.basename(os.path.dirname(img['file_name']))) ulti.make_dir(fullpath) path = os.path.join( fullpath, os.path.splitext(os.path.basename(img['file_name']))[0] + '.json') ulti.write_json(annotations, path)
import os import ulti import tqdm from shutil import copyfile if __name__ == "__main__": ulti.copy_model() info = ulti.load_json() video = info['annotated_video'] input_path = os.path.join(info['training_dir'], video, 'inference', 'ro_bdd_test_cocostyle', 'bbox.json') dataset_dir = info['dataset_dir'] output_dir = os.path.join(dataset_dir, info['experiment'], 'Raw_Detection') ulti.make_dir(output_dir) copyfile(input_path, os.path.join(output_dir, 'bbox.json')) prediction = ulti.load_json(input_path) category = ulti.load_json( os.path.join(dataset_dir, 'Categories', 'Road_Objects.json')) dataset = ulti.load_json(os.path.join(dataset_dir, 'Info', video + '.json')) list_img_id_dataset = [] list_img_id_prediction = [] ann_id = 0 for ann in prediction: list_img_id_prediction.append(ann['image_id'])
import os import ulti import shutil if __name__ == "__main__": ulti.copy_model() info = ulti.load_json() training_dir = info['training_dir'] dataset_name = info['annotated_video'] dir_input = os.path.join(training_dir, dataset_name) if os.path.exists(dir_input): shutil.rmtree(dir_input) src = os.path.join( info['dataset_dir'], '../Initial_model/e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml') dst = os.path.join(info['training_dir'], 'e2e_faster_rcnn_R_50_FPN_Xconv1fc_1x_gn.yaml') shutil.copyfile(src, dst)
def main(videonames=[]): info = ulti.load_json() if not videonames: videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) output_kf_bbox = False if info['annotated_video'] == info['dataset_name']: tq = tqdm.tqdm(total=len(videonames)) for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) tracker = DeepSortTracker(output_kf_bbox=output_kf_bbox) tracker.track(os.path.join(info['dataset_dir'], 'Images', videoname), os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', videoname), None, os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort', 'Json', videoname), visualization=False, tq_display=False, reverse=False) else: videoname = info['annotated_video'] tracker = DeepSortTracker(output_kf_bbox=output_kf_bbox) tracker.track(os.path.join(info['dataset_dir'], 'Images', videoname), os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', videoname), None, os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort', 'Json', videoname), visualization=False, tq_display=True, reverse=False) if info['annotated_video'] == info['dataset_name']: tq = tqdm.tqdm(total=len(videonames)) for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) tracker = DeepSortTracker(output_kf_bbox=output_kf_bbox) tracker.track(os.path.join(info['dataset_dir'], 'Images', videoname), os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', videoname), None, os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort_Reverse', 'Json', videoname), visualization=False, tq_display=False, reverse=True) else: videoname = info['annotated_video'] tracker = DeepSortTracker(output_kf_bbox=output_kf_bbox) tracker.track(os.path.join(info['dataset_dir'], 'Images', videoname), os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', videoname), None, os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort_Reverse', 'Json', videoname), visualization=False, tq_display=True, reverse=True)
def create_train(): info = ulti.load_json() dir_input = info['dataset_dir'] dir_output = os.path.join(info['dataset_dir'], 'RCNN_data') video = info['annotated_video'] only_use_true_gt = False path = os.path.join(dir_input, 'Info', video + '.json') dataset = ulti.load_json(path) images = dataset['images'] videos = dataset['videos'] categories = ulti.load_json(dir_input + '/Categories/Road_Objects.json') categories = categories['category'] dataset = { 'categories': categories, 'annotations': [], 'videos': [], 'images': [] } video_names = [] video_ids = [] for vid in videos: dataset['videos'].append(vid) video_names.append(vid['name']) video_ids.append(vid['id']) dict_images = {} for image in images: image['file_name'] = image['file_name'].replace('Images\\\\', '') image['file_name'] = image['file_name'].replace('Images/', '') image['file_name'] = image['file_name'].replace('Images\\', '') key = os.path.join( os.path.basename(os.path.dirname(image['file_name'])), os.path.splitext(os.path.basename(image['file_name']))[0]) dict_images[key] = image list_images = [] annotations = [] ins_id = 0 tq = tqdm.tqdm(total=len(video_names)) for id, video in zip(video_ids, video_names): # print(video) tq.update(1) dir_tubelet = os.path.join(info['dataset_dir'], info['experiment'], 'Add_instances', 'Tubelet', video + '.json') with open(dir_tubelet) as f: tubelet = json.load(f) tubelet_by_frame = tubelet['tubelet_by_frame'] tubelet_by_id = tubelet['tubelet_by_id'] for filename in tubelet_by_frame.keys(): tubelet = tubelet_by_frame[filename] if len(tubelet.keys()) > 0: image = dict_images[os.path.join(video, filename)] if (image['has_gt'] and only_use_true_gt) or not only_use_true_gt: list_images.append(image) for key in tubelet.keys(): ann = tubelet[key] ann['bbox'] = [ ann['bbox']['x'], ann['bbox']['y'], ann['bbox']['w'], ann['bbox']['h'] ] ann['id'] = ins_id ann['image_id'] = image['id'] ins_id += 1 annotations.append(ann) tq.set_description('Video {}'.format( os.path.join(video, filename))) dataset['annotations'] = annotations dataset['videos'] = videos dataset['images'] = list_images print('train: ', len(list_images)) ulti.make_dir(dir_output) outfile = ulti.write_json(dataset, file=os.path.join(dir_output, 'train.json'))
def main(model_path=''): ulti.copy_model(model_path=model_path) info = ulti.load_json() video = info['annotated_video'] input_path = os.path.join(info['training_dir'], video, 'inference', 'ro_bdd_test_cocostyle', 'bbox.json') dataset_dir = info['dataset_dir'] output_dir = os.path.join(dataset_dir, info['experiment'], 'Raw_Detection') ulti.make_dir(output_dir) copyfile(input_path, os.path.join(output_dir, 'bbox.json')) prediction = ulti.load_json(input_path) category = ulti.load_json( os.path.join(dataset_dir, 'Categories', 'Road_Objects.json')) dataset = ulti.load_json(os.path.join(dataset_dir, 'Info', video + '.json')) list_img_id_dataset = [] list_img_id_prediction = [] ann_id = 0 for ann in prediction: list_img_id_prediction.append(ann['image_id']) tq = tqdm.tqdm(total=len(dataset['images'])) for img in dataset['images']: list_img_id_dataset.append(img['id']) tq.set_description('Video Frame {}'.format(img['id'])) tq.update(1) index = [ i for i, x in enumerate(list_img_id_prediction) if x == img['id'] ] if len(index) > 0: # print(img['id'], index) annotations = [] for ind in index: ann = {} ann['id'] = ann_id ann['image_id'] = prediction[ind]['image_id'] ann['track_id'] = -1 ann['category_id'] = prediction[ind]['category_id'] ann['score'] = prediction[ind]['score'] ann['iscrowd'] = 0 # always 0 ann['bbox'] = { 'x': int(prediction[ind]['bbox'][0]), 'y': int(prediction[ind]['bbox'][1]), 'w': int(prediction[ind]['bbox'][2]), 'h': int(prediction[ind]['bbox'][3]) } ann['trackable'] = True ann['segmentation'] = [] ann['area'] = 0 annotations.append(ann) ann_id += 1 if len(annotations) > 0: fullpath = os.path.join( dataset_dir, 'Annotations', category['name'], os.path.basename(os.path.dirname(img['file_name']))) ulti.make_dir(fullpath) fullpath = os.path.join( output_dir, 'Raw', 'Json', os.path.basename(os.path.dirname(img['file_name']))) ulti.make_dir(fullpath) path = os.path.join( fullpath, os.path.splitext(os.path.basename(img['file_name']))[0] + '.json') ulti.write_json(annotations, path)
def run_trackers(): info = ulti.load_json() videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) tq = tqdm.tqdm(total=len(videonames)) alpha = 0.5 frequency_threshold = 5 iou_threshold = 0.3 iou_threshold2 = 0.8 border = 8 for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) dir_tubelet = os.path.join(info['dataset_dir'], info['experiment'], 'Filter_data', 'Tubelet', videoname + '.json') ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Add_instances', 'Videos', videoname)) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Add_instances', 'Tubelet')) with open(dir_tubelet) as f: tubelet = json.load(f) tubelet_by_frame = tubelet['tubelet_by_frame'] tubelet_by_id = tubelet['tubelet_by_id'] filenames = os.listdir(os.path.join(info['dataset_dir'], 'Images', videoname)) filenames = sorted(filenames) frames = [] for i, frame in enumerate(filenames): frame = os.path.splitext(frame)[0] frames.append(frame) img = cv2.imread(os.path.join(info['dataset_dir'], 'Images', videoname, filenames[0])) height, width, channels = img.shape for id in list(tubelet_by_id.keys()): tubelet = tubelet_by_id[id] if len(tubelet.keys()) >= frequency_threshold: keys = sorted(list(tubelet.keys())) # forward process prev_idx = frames.index(keys[0]) tracker = None for idx in range(frames.index(keys[1]), len(frames)): if frames[idx] not in keys: # begin tracking if tracker is None and frames[prev_idx] in tubelet.keys(): prev_bbox = tubelet[frames[prev_idx]]['bbox'] if prev_bbox['x'] >= width / border and (prev_bbox['x'] + prev_bbox['w']) <= width * ( border - 1) / border: tracker = init_tracker( init_image_file=os.path.join(info['dataset_dir'], 'Images', videoname, filenames[prev_idx]), init_rect=[prev_bbox['x'], prev_bbox['y'], prev_bbox['w'], prev_bbox['h']], device='cuda') prev_rect = [prev_bbox['x'], prev_bbox['y'], prev_bbox['w'], prev_bbox['h']] iou = -1 if tracker: rect, tracker = run_tracker( [os.path.join(info['dataset_dir'], 'Images', videoname, filenames[idx])], tracker, visualization=False) rect = rect[0] if iou < 0 or iou >= iou_threshold: iou = bb_intersection_over_union(rect, prev_rect) prev_rect = rect if os.path.splitext(filenames[idx])[0] in tubelet_by_frame.keys(): other_boxes = tubelet_by_frame[os.path.splitext(filenames[idx])[0]] other_iou = 0 for other_key in other_boxes.keys(): other_box = other_boxes[other_key]['bbox'] other_rect = [other_box['x'], other_box['y'], other_box['w'], other_box['h']] other_iou = bb_intersection_over_union(rect, other_rect) if other_iou >= iou_threshold2: break if other_iou >= iou_threshold2: break if iou >= iou_threshold: tubelet[frames[idx]] = copy.deepcopy(tubelet[list(tubelet.keys())[0]]) tubelet[frames[idx]]['bbox'] = {'x': rect[0], 'y': rect[1], 'w': rect[2], 'h': rect[3]} tubelet[frames[idx]]['track_bbox_xyxy'] = xywh_to_xyxy(rect) tubelet[frames[idx]]['add_by_tracker'] = True if frames[idx] not in tubelet_by_frame.keys(): tubelet_by_frame[frames[idx]] = {} tubelet_by_frame[frames[idx]][id] = tubelet[frames[idx]] else: break else: # stop tracking if tracker: tracker = None prev_idx = idx ulti.write_json({'tubelet_by_frame': tubelet_by_frame, 'tubelet_by_id': tubelet_by_id}, os.path.join(info['dataset_dir'], info['experiment'], 'Add_instances', 'Tubelet', videoname + '.json')) for frame in tubelet_by_frame.keys(): tubelet = tubelet_by_frame[frame] ulti.write_json(tubelet, os.path.join(info['dataset_dir'], info['experiment'], 'Add_instances', 'Videos', videoname, frame + '.json'))
def init_tracklet(): info = ulti.load_json() ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Tracklet')) videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) tq = tqdm.tqdm(total=len(videonames)) for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) dir_track = os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort', 'Json', videoname) dir_track_reverse = os.path.join(info['dataset_dir'], info['experiment'], 'Track', 'DeepSort_Reverse', 'Json', videoname) tracklet_by_id = {} tracklet_by_frame = {} list_images = os.listdir(dir_track) list_images = sorted(list_images) for i, image in enumerate(list_images): filename = os.path.splitext(image)[0] if filename not in tracklet_by_frame.keys(): tracklet_by_frame[filename] = {} with open(os.path.join(dir_track, image)) as f: data = json.load(f) for x in data: tracklet_by_frame[filename][x['id']] = {} tracklet_by_frame[filename][x['id']]['track_bbox_xyxy'] = x['bbox_xyxy'] if x['id'] not in tracklet_by_id.keys(): tracklet_by_id[x['id']] = {} tracklet_by_id[x['id']][filename] = {} tracklet_by_id[x['id']][filename]['track_bbox_xyxy'] = x['bbox_xyxy'] tracklet_by_id = dict(sorted(tracklet_by_id.items())) for key in tracklet_by_id.keys(): tracklet_by_id[key] = dict(sorted(tracklet_by_id[key].items())) tracklet_by_frame = dict(sorted(tracklet_by_frame.items())) for key in tracklet_by_frame.keys(): tracklet_by_frame[key] = dict(sorted(tracklet_by_frame[key].items())) tracklet_by_id2 = {} tracklet_by_frame2 = {} list_reverse_images = os.listdir(dir_track_reverse) list_reverse_images = sorted(list_reverse_images) for i, image in enumerate(list_reverse_images): filename = os.path.splitext(image)[0] if filename not in tracklet_by_frame2.keys(): tracklet_by_frame2[filename] = {} with open(os.path.join(dir_track_reverse, image)) as f: data = json.load(f) for x in data: tracklet_by_frame2[filename][x['id']] = {} tracklet_by_frame2[filename][x['id']]['track_bbox_xyxy'] = x['bbox_xyxy'] if x['id'] not in tracklet_by_id2.keys(): tracklet_by_id2[x['id']] = {} tracklet_by_id2[x['id']][filename] = {} tracklet_by_id2[x['id']][filename]['track_bbox_xyxy'] = x['bbox_xyxy'] if image in list_images: break tracklet_by_id2 = dict(sorted(tracklet_by_id2.items())) for key in tracklet_by_id2.keys(): tracklet_by_id2[key] = dict(sorted(tracklet_by_id2[key].items())) tracklet_by_frame2 = dict(sorted(tracklet_by_frame2.items())) for key in tracklet_by_frame2.keys(): tracklet_by_frame2[key] = dict(sorted(tracklet_by_frame2[key].items())) if len(list(tracklet_by_frame.keys())) > 0 and len(list(tracklet_by_frame2.keys())) > 0: boxes1 = np.zeros((len(tracklet_by_frame[list(tracklet_by_frame.keys())[0]]), 4)) for i, box in enumerate(list(tracklet_by_frame[list(tracklet_by_frame.keys())[0]].items())): boxes1[i, :] = box[1]['track_bbox_xyxy'] boxes2 = np.zeros((len(tracklet_by_frame2[list(tracklet_by_frame2.keys())[-1]]), 4)) for i, box in enumerate(list(tracklet_by_frame2[list(tracklet_by_frame2.keys())[-1]].items())): boxes2[i, :] = box[1]['track_bbox_xyxy'] iou = torchvision.ops.box_iou(torch.from_numpy(boxes1), torch.from_numpy(boxes2)) iou = iou.detach().numpy() iou_threshold = 0.5 iou[iou < iou_threshold] = 0 max_iou = 1 while max_iou != 0: max_iou = np.amax(iou) if max_iou == 0: break (i, j) = np.where(iou == max_iou) i = i[0] j = j[0] # i boxes1 tracklet_by_frame[list(tracklet_by_frame.keys())[0]] # j boxes2 tracklet_by_frame2[list(tracklet_by_frame2.keys())[-1]] id1 = list(tracklet_by_frame[list(tracklet_by_frame.keys())[0]].keys())[i] id2 = list(tracklet_by_frame2[list(tracklet_by_frame2.keys())[-1]].keys())[j] # if videoname == '000017' and id1 == 4: # print(videoname, max_iou, id1, id2) for k in tracklet_by_id2[id2].keys(): tracklet_by_id[id1][k] = tracklet_by_id2[id2][k] tracklet_by_id2[id2][k] = None for k in tracklet_by_frame2.keys(): if k not in tracklet_by_frame.keys(): tracklet_by_frame[k] = {} if id2 in tracklet_by_frame2[k].keys(): tracklet_by_frame[k][id1] = tracklet_by_frame2[k][id2] tracklet_by_frame2[k][id2] = None iou[i, :] = 0 iou[:, j] = 0 tracklet_by_id = dict(sorted(tracklet_by_id.items())) for key in tracklet_by_id.keys(): tracklet_by_id[key] = dict(sorted(tracklet_by_id[key].items())) tracklet_by_frame = dict(sorted(tracklet_by_frame.items())) for key in tracklet_by_frame.keys(): tracklet_by_frame[key] = dict(sorted(tracklet_by_frame[key].items())) ulti.write_json({'tracklet_by_id': tracklet_by_id, 'tracklet_by_frame': tracklet_by_frame}, file=os.path.join(info['dataset_dir'], info['experiment'], 'Tracklet', videoname + '.json'))
def create_tubelet(): info = ulti.load_json() ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Tubelet')) videonames = os.listdir(os.path.join(info['dataset_dir'], 'Images')) videonames = sorted(videonames) tq = tqdm.tqdm(total=len(videonames)) for videoname in videonames: tq.set_description('Video {}'.format(videoname)) tq.update(1) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Videos', videoname)) ulti.make_dir(os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Visualization', videoname)) dir_tracklet = os.path.join(info['dataset_dir'], info['experiment'], 'Tracklet', videoname + '.json') dir_detection = os.path.join(info['dataset_dir'], info['experiment'], 'Detection', 'Json', videoname) with open(dir_tracklet) as f: tracklet_data = json.load(f) tracklet_by_id = tracklet_data['tracklet_by_id'] tracklet_by_frame = tracklet_data['tracklet_by_frame'] tubelet_by_frame = {} tubelet_by_id = {} for frame in tracklet_by_frame.keys(): tubelet_data = [] with open(os.path.join(dir_detection, frame + '.json')) as f: detection_data = json.load(f) detection_bbox = np.zeros((len(detection_data), 4)) for i, box in enumerate(detection_data): detection_bbox[i, 0] = box['bbox']['x'] detection_bbox[i, 1] = box['bbox']['y'] detection_bbox[i, 2] = box['bbox']['x'] + box['bbox']['w'] detection_bbox[i, 3] = box['bbox']['y'] + box['bbox']['h'] track_bbox = np.zeros((len(tracklet_by_frame[frame]), 4)) for i, key in enumerate(tracklet_by_frame[frame].keys()): box = tracklet_by_frame[frame][key] track_bbox[i, :] = box['track_bbox_xyxy'] if detection_bbox.shape[0] == 0 or track_bbox.shape[0] == 0: continue iou = torchvision.ops.box_iou(torch.from_numpy(detection_bbox), torch.from_numpy(track_bbox)) iou = iou.detach().numpy() iou_threshold = 0.7 iou[iou < iou_threshold] = 0 max_iou = 1 while max_iou != 0: max_iou = np.amax(iou) if max_iou == 0: break (i, j) = np.where(iou == max_iou) i = i[0] j = j[0] detection_data[i]['track_id'] = list(tracklet_by_frame[frame].keys())[j] detection_data[i]['track_bbox_xyxy'] = tracklet_by_frame[frame][detection_data[i]['track_id']]['track_bbox_xyxy'] tubelet_data.append(detection_data[i]) iou[i, :] = 0 iou[:, j] = 0 if len(tubelet_data) > 0: ulti.write_json(tubelet_data, os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Videos', videoname, frame + '.json')) tubelet_by_frame[frame] = {} for x in tubelet_data: tubelet_by_frame[frame][x['track_id']] = x if x['track_id'] not in tubelet_by_id: tubelet_by_id[x['track_id']] = {} tubelet_by_id[x['track_id']][frame] = x ulti.write_json({'tubelet_by_frame': tubelet_by_frame, 'tubelet_by_id': tubelet_by_id}, os.path.join(info['dataset_dir'], info['experiment'], 'Tubelet', 'Tubelet', videoname + '.json'))