def task1_1(save_path=None): reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) # add probability to delete bounding boxes drop_values = np.linspace(0, 1, 11) maps = [] for drop in drop_values: noise_params = {'drop': drop, 'mean': 0, 'std': 0} gt_noisy = reader.get_annotations(classes=['car'], noise_params=noise_params) y_true = [] y_pred = [] for frame in gt.keys(): y_true.append(gt.get(frame)) y_pred.append(gt_noisy.get(frame, [])) map, _, _ = mean_average_precision(y_true, y_pred) maps.append(map) plt.plot(drop_values, maps) plt.xticks(drop_values) plt.xlabel('drop prob') plt.ylabel('mAP') plt.show() if save_path is not None: plt.savefig(os.path.join(save_path, 'map_drop_bbox.png')) # add noise to the size and position of bounding boxes std_values = np.linspace(0, 100, 11) maps = [] for std in std_values: noise_params = {'drop': 0, 'mean': 0, 'std': std} gt_noisy = reader.get_annotations(classes=['car'], noise_params=noise_params) y_true = [] y_pred = [] for frame in gt.keys(): y_true.append(gt.get(frame)) y_pred.append(gt_noisy.get(frame, [])) map, _, _ = mean_average_precision(y_true, y_pred) maps.append(map) plt.xlabel('std') plt.ylabel('mAP') plt.xticks(std_values) plt.plot(std_values, maps) plt.show() if save_path is not None: plt.savefig(os.path.join(save_path, 'map_noisy_bbox.png'))
def task2_2(debug=False, det_path='data/AICity_data/train/S03/c010/det/det_mask_rcnn.txt'): """ Object tracking: tracking with a Kalman filter """ reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) reader = AICityChallengeAnnotationReader(path=det_path) dets = reader.get_annotations(classes=['car']) cap = cv2.VideoCapture('data/AICity_data/train/S03/c010/vdo.avi') tracker = Sort() tracks = defaultdict(list) y_true = [] y_pred = [] acc = MOTAcumulator() for frame in dets.keys(): detections = dets.get(frame, []) new_detections = tracker.update( np.array([[*d.bbox, d.score] for d in detections])) new_detections = [ Detection(frame, int(d[-1]), 'car', *d[:4]) for d in new_detections ] y_true.append(gt.get(frame, [])) y_pred.append(new_detections) acc.update(y_true[-1], y_pred[-1]) if debug: cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() for d in new_detections: tracks[d.id].append(d.bbox) np.random.seed(d.id) color = tuple(np.random.randint(0, 256, 3).tolist()) for dd in tracks[d.id]: cv2.circle(img, (int( (dd[0] + dd[2]) / 2), int((dd[1] + dd[3]) / 2)), 5, color, -1) cv2.imshow('image', cv2.resize(img, (900, 600))) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car']) idf1, idp, idr = acc.get_idf1() print( f"AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}, IDF1: {idf1:.4f}, IDP: {idp:.4f}, IDR: {idr:.4f}" )
def evaluate(model, data_loader, device, save_path=None): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' y_true = [] y_pred = [] for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time metric_logger.update(model_time=model_time) for target, output in zip(targets, outputs): frame = target['image_id'].item() y_true.append([ Detection(frame, None, label, *box) for box, label in zip(target['boxes'], target['labels']) ]) y_pred.append([ Detection(frame, None, label, *box, score) for box, label, score in zip(output['boxes'], output['labels'], output['scores']) ]) evaluator_time = time.time() map, _, _ = mean_average_precision(y_true, y_pred, sort_method='score') evaluator_time = time.time() - evaluator_time metric_logger.update(map=map, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) torch.set_num_threads(n_threads) if save_path: os.makedirs(os.path.dirname(save_path), exist_ok=True) with open(save_path, 'w') as f: for frame_dets in y_pred: for d in frame_dets: f.write( f'{d.frame}, -1, {d.xtl}, {d.ytl}, {d.width}, {d.height}, {d.score}, -1, -1, -1\n' )
def task1_2(): reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) for detector in ['mask_rcnn', 'ssd512', 'yolo3']: reader = AICityChallengeAnnotationReader( path=f'data/AICity_data/train/S03/c010/det/det_{detector}.txt') det = reader.get_annotations(classes=['car']) y_true = [] y_pred = [] for frame in gt.keys(): y_true.append(gt.get(frame)) y_pred.append(det.get(frame, [])) map, _, _ = mean_average_precision(y_true, y_pred) print(f'{detector} mAP: {map:.4f}')
def task4(adaptive, random_search, color_space, channels, model_frac=0.25, save_path=None, min_width=120, max_width=800, min_height=100, max_height=600, debug=0): """ Color modelling """ n_ch = len(channels) # Read information reader = AICityChallengeAnnotationReader( path='data/AICity_data/train/S03/c010/gt/gt.txt') gt = reader.get_annotations(classes=['car'], only_not_parked=True) roi = cv2.imread('data/AICity_data/train/S03/c010/roi.jpg', cv2.IMREAD_GRAYSCALE) # Model Background bg_model = SingleGaussianBackgroundModel( video_path='data/AICity_data/train/S03/c010/vdo.avi', color_space=color_space, channels=channels, resize=None) video_length = bg_model.length bg_model.fit(start=0, length=int(video_length * 0.25)) # Video length start_frame = int(video_length * model_frac) end_frame = int(video_length) # hyperparameter search if random_search: alphas = np.random.choice(np.linspace(2, 4, 50), 25) rhos = np.random.choice(np.linspace(0.001, 0.1, 50), 25) if adaptive else [0] combinations = [(alpha, rho) for alpha, rho in zip(alphas, rhos)] else: alphas = [3.5] rhos = [0.005] if adaptive else [0] combinations = [(alpha, rho) for alpha in alphas for rho in rhos] for alpha, rho in combinations: y_true = [] y_pred = [] if save_path: gif_name = f'100_task3_alpha_{str(alpha)}_rho_{str(rho)}_color_{color_space}_channels_{str(n_ch)}_{time.time()}.gif' writer = imageio.get_writer(os.path.join(save_path, gif_name), fps=10) for frame in trange( start_frame, end_frame, desc= f'obtaining foreground and detecting objects. Alpha {alpha} Rho {rho}' ): if frame == 635: break frame_img, mask, _ = bg_model.evaluate(frame=frame, alpha=alpha) mask = mask & roi non_post_mask = mask mask = postprocess(mask) detections = bounding_boxes(mask, min_height, max_height, min_width, max_width, frame) annotations = gt.get(frame, []) if save_path: img = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) for det in detections: cv2.rectangle(img, (det.xtl, det.ytl), (det.xbr, det.ybr), (0, 255, 0), 3) for det in annotations: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), (0, 0, 255), 2) writer.append_data(img) if debug >= 1: shape = (480, 270) cv2.imshow(f'BGR Image', cv2.resize(img, shape)) cv2.imshow(f'Segmentation using {color_space}', cv2.resize(non_post_mask, shape)) cv2.imshow(f'Segmentation Morphed using {color_space}', cv2.resize(mask, shape)) if cv2.waitKey(1) & 0xFF == ord('q'): break y_pred.append(detections) y_true.append(annotations) cv2.destroyAllWindows() if save_path: writer.close() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car']) print(f'alpha: {alpha:.1f}, rho: {rho:.3f}, AP: {ap:.4f}') print(f'prec: {prec:.4f}, Recall {rec:.4f}')
def task1_2(adaptive, random_search, model_frac=0.25, min_width=120, max_width=800, min_height=100, max_height=600, debug=0, save_path=None): reader = AICityChallengeAnnotationReader( path='data/AICity_data/train/S03/c010/gt/gt.txt') gt = reader.get_annotations(classes=['car'], only_not_parked=True) roi = cv2.imread('data/AICity_data/train/S03/c010/roi.jpg', cv2.IMREAD_GRAYSCALE) bg_model = SingleGaussianBackgroundModel( video_path='data/AICity_data/train/S03/c010/vdo.avi') video_length = bg_model.length bg_model.fit(start=0, length=int(video_length * model_frac)) start_frame = int(video_length * model_frac) end_frame = video_length # hyperparameter search if random_search: alphas = np.random.choice(np.linspace(2, 4, 50), 25) rhos = np.random.choice(np.linspace(0.001, 0.1, 50), 25) if adaptive else [0] combinations = [(alpha, rho) for alpha, rho in zip(alphas, rhos)] else: alphas = [2, 2.5, 3, 3.5, 4] rhos = [0.005, 0.01, 0.025, 0.05, 0.1] if adaptive else [0] combinations = [(alpha, rho) for alpha in alphas for rho in rhos] for alpha, rho in combinations: if save_path: writer = imageio.get_writer(os.path.join( save_path, f'task1_2_alpha{alpha:.1f}_rho{rho:.3f}.gif'), fps=10) y_true = [] y_pred = [] for frame in trange(start_frame, end_frame, desc='evaluating frames'): _, mask, _ = bg_model.evaluate(frame=frame, alpha=alpha, rho=rho) mask = mask & roi mask = postprocess(mask) detections = bounding_boxes(mask, min_height, max_height, min_width, max_width, frame) annotations = gt.get(frame, []) if debug >= 1 or save_path: img = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) for det in detections: cv2.rectangle(img, (det.xtl, det.ytl), (det.xbr, det.ybr), (0, 255, 0), 2) for det in annotations: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), (0, 0, 255), 2) if save_path: writer.append_data(img) if debug >= 1: cv2.imshow('result', img) if cv2.waitKey(1) & 0xFF == ord('q'): break y_pred.append(detections) y_true.append(annotations) cv2.destroyAllWindows() if save_path: writer.close() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car']) print(f'alpha: {alpha:.1f}, rho: {rho:.3f}, AP: {ap:.4f}')
def task3(methods, model_frac=0.25, min_width=120, max_width=800, min_height=100, max_height=600, save_path=None, debug=0): """ Comparison with the state of the art """ reader = AICityChallengeAnnotationReader( path='data/AICity_data/train/S03/c010/gt/gt.txt') gt = reader.get_annotations(classes=['car'], only_not_parked=True) roi = cv2.imread('data/AICity_data/train/S03/c010/roi.jpg', cv2.IMREAD_GRAYSCALE) cap = cv2.VideoCapture('data/AICity_data/train/S03/c010/vdo.avi') video_length = cap.get(cv2.CAP_PROP_FRAME_COUNT) start_frame = int(video_length * model_frac) end_frame = int(video_length) for method in methods: backSub = sota_bg_subtractor(method) for _ in trange(start_frame, desc='modelling background'): ret, img = cap.read() backSub.apply(img) if save_path: writer = imageio.get_writer(os.path.join( save_path, f'task3_method_' + method + '.gif'), fps=10) y_pred = [] y_true = [] for frame in trange(start_frame, end_frame, desc='evaluating frames'): cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() mask = backSub.apply(img) mask = mask & roi mask = postprocess(mask) detections = bounding_boxes(mask, min_height, max_height, min_width, max_width, frame) annotations = gt.get(frame, []) if debug >= 1 or save_path: img = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) for det in detections: cv2.rectangle(img, (det.xtl, det.ytl), (det.xbr, det.ybr), (0, 255, 0), 2) for det in annotations: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), (0, 0, 255), 2) if save_path: writer.append_data(img) elif debug == 1: cv2.imshow('result', img) if cv2.waitKey(1) & 0xFF == ord('q'): break y_pred.append(detections) y_true.append(annotations) cv2.destroyAllWindows() if save_path: writer.close() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car']) print( f'Method: {method}, AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}' )
def task1_1(architecture, start=0, length=None, save_path='results/week3', gpu=0, visualize=False, save_detection='detection_results/'): """ Object detection: off-the-shelf """ tensor = transforms.ToTensor() if architecture.lower() == 'fasterrcnn': model = detection.fasterrcnn_resnet50_fpn(pretrained=True) elif architecture.lower() == 'maskrcnn': model = detection.maskrcnn_resnet50_fpn(pretrained=True) else: raise ValueError(architecture) save_path = os.path.join(save_path, architecture) # Read Video and prepare ground truth cap = cv2.VideoCapture('data/AICity_data/train/S03/c010/vdo.avi') if not length: length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) gt = {frame: gt[frame] for frame in range(start, start + length)} # Start Inference device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) model.to(device) model.eval() detections = {} y_true, y_pred = [], [] if save_detection: path = os.path.join(save_detection, architecture) if not os.path.exists(path): os.makedirs(path) detection_file = open(f'{path}/{architecture.lower()}.txt', 'w') with torch.no_grad(): for frame in range(start, length): cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() # Transform input to tensor print(f'Predict: {frame}') start_t = time.time() x = [tensor(img).to(device)] preds = model(x)[0] print( f'Inference time per frame: {round(time.time() - start_t, 2)}') # filter car predictions and confidences joint_preds = list( zip(preds['labels'], preds['boxes'], preds['scores'])) car_det = list(filter(lambda x: x[0] == 3, joint_preds)) # car_det = list(filter(lambda x: x[2] > 0.70, car_det)) car_det = get_nms(car_det, 0.7) # add detections detections[frame] = [] for det in car_det: det_obj = Detection(frame=frame, id=None, label='car', xtl=float(det[1][0]), ytl=float(det[1][1]), xbr=float(det[1][2]), ybr=float(det[1][3]), score=det[2]) detections[frame].append(det_obj) if save_detection: detection_file.write( f"{frame},-1,{det_obj.xtl},{det_obj.ytl},{det_obj.width},{det_obj.height},{det_obj.score},-1,-1,-1\n" ) y_pred.append(detections[frame]) y_true.append(gt.get(frame, [])) ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car']) print( f'Network: {architecture}, AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}' ) if visualize: print(f'Saving result to {save_path}') if not os.path.exists(save_path): os.makedirs(save_path) video_iou_plot(gt, detections, video_path='data/AICity_data/train/S03/c010/vdo.avi', title=f'{architecture} detections', save_path=save_path) cv2.destroyAllWindows() if save_detection: detection_file.close()
def task2_1(debug=False, save_path=None, det_path='data/AICity_data/train/S03/c010/det/det_mask_rcnn.txt'): """ Object tracking: tracking by overlap """ cap = cv2.VideoCapture('data/AICity_data/train/S03/c010/vdo.avi') reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) reader = AICityChallengeAnnotationReader(path=det_path) dets = reader.get_annotations(classes=['car']) if save_path: writer = imageio.get_writer(os.path.join(save_path, f'task21.gif'), fps=10) accumulator = MOTAcumulator() y_true = [] y_pred = [] y_pred_refined = [] tracks = [] max_track = 0 for frame in dets.keys(): if debug or save_path: cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() detections_on_frame = dets.get(frame, []) tracks, frame_tracks, max_track = update_tracks_by_overlap( tracks, detections_on_frame, max_track) frame_detections = [] for track in frame_tracks: det = track.last_detection() frame_detections.append(det) if debug or save_path: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), track.color, 2) cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ytl) - 15), track.color, -2) cv2.putText(img, str(det.id), (int(det.xtl), int(det.ytl)), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2) for dd in track.detections: cv2.circle(img, dd.center, 5, track.color, -1) y_pred_refined.append(frame_detections) y_pred.append(detections_on_frame) y_true.append(gt.get(frame, [])) accumulator.update(y_true[-1], y_pred_refined[-1]) if save_path: writer.append_data(cv2.resize(img, (600, 350))) elif debug: cv2.imshow('result', cv2.resize(img, (900, 600))) if cv2.waitKey(1) & 0xFF == ord('q'): break cv2.destroyAllWindows() if save_path: writer.close() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car'], sort_method='score') print(f'Original AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}') ap, prec, rec = mean_average_precision(y_true, y_pred_refined, classes=['car'], sort_method='score') print( f'After refinement AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}' ) print('\nAdditional metrics:') print(accumulator.get_idf1())
def launch_test_kalman_filter(save_path, distance_thresholds, min_track_len, min_width, min_height, sequence, camera, detector): save_video = False save_summary = False fps = 24 os.makedirs(save_path, exist_ok=True) reader = AICityChallengeAnnotationReader(path='data/AICity_data/train/' + sequence + '/' + camera + '/gt/gt.txt') gt = reader.get_annotations(classes=['car']) reader = AICityChallengeAnnotationReader(path='data/AICity_data/train/' + sequence + '/' + camera + '/det/det_' + detector + '.txt') dets = reader.get_annotations(classes=['car']) cap = cv2.VideoCapture('data/AICity_data/train/' + sequence + '/' + camera + '/vdo.avi') n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) if save_video: writer = imageio.get_writer(os.path.join( save_path, 'task1_' + sequence + '_' + camera + '_' + detector + '.gif'), fps=fps) tracker = Sort() y_true = [] tracks = [] max_track = 0 video_percentage = 1 start = 0 end = int(n_frames * video_percentage) for frame in trange(start, end, desc='Tracking'): detections_on_frame_ = dets.get(frame, []) detections_on_frame = [] for d in detections_on_frame_: if min_width < (d.ybr - d.ytl) and min_height < (d.xbr - d.xtl): detections_on_frame.append(d) detections_on_frame = tracker.update( np.array([[*d.bbox, d.score] for d in detections_on_frame])) detections_on_frame = [ Detection(frame, int(d[-1]), 'car', *d[:4]) for d in detections_on_frame ] tracks, frame_tracks, max_track = update_tracks_by_overlap( tracks, detections_on_frame, max_track, refinement=False, optical_flow=None) y_true.append(gt.get(frame, [])) idf1s = [] for distance_threshold in distance_thresholds: accumulator = MOTAcumulator() y_pred = [] moving_tracks = remove_static_tracks(tracks, distance_threshold, min_track_len) detections = [] for track in moving_tracks: detections.extend(track.detections) detections = group_by_frame(detections) for frame in trange(start, end, desc='Accumulating detections'): if save_video: cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() for det in y_true[frame]: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), (0, 255, 0), 6) frame_detections = [] for det in detections.get(frame, []): frame_detections.append(det) if save_video: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), track.color, 6) cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ytl) - 15), track.color, -6) cv2.putText(img, str(det.id), (int(det.xtl), int(det.ytl)), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 6) cv2.circle(img, track.detections[-1].center, 5, track.color, -1) y_pred.append(frame_detections) if save_video: writer.append_data(cv2.resize(img, (600, 350))) accumulator.update(y_true[frame], y_pred[-1]) ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car'], sort_method=None) print(f'AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}') print('Additional metrics:') summary = accumulator.get_idf1() # As mentioned in https://github.com/cheind/py-motmetrics: # FAR = FalsePos / Frames * 100 # MOTP = (1 - MOTP) * 100 print(summary) if save_summary: with open( os.path.join( save_path, 'task1_' + sequence + '_' + camera + '_' + detector + '_' + str(distance_threshold) + '.txt'), 'w') as f: f.write(str(summary)) idf1s.append(summary['idf1']['acc'] * 100) cv2.destroyAllWindows() if save_video: writer.close() return idf1s
def launch_test_optical_flow(save_path, distance_thresholds, min_track_len, min_width, min_height, sequence, camera, detector): save_video = False save_summary = False fps = 24 os.makedirs(save_path, exist_ok=True) reader = AICityChallengeAnnotationReader(path='data/AICity_data/train/' + sequence + '/' + camera + '/gt/gt.txt') gt = reader.get_annotations(classes=['car']) reader = AICityChallengeAnnotationReader(path='data/AICity_data/train/' + sequence + '/' + camera + '/det/det_' + detector + '.txt') dets = reader.get_annotations(classes=['car']) cap = cv2.VideoCapture('data/AICity_data/train/' + sequence + '/' + camera + '/vdo.avi') n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) if save_video: writer = imageio.get_writer(os.path.join( save_path, 'task1_' + sequence + '_' + camera + '_' + detector + '.gif'), fps=fps) y_true = [] tracks = [] max_track = 0 previous_frame = None video_percentage = 1 start = 0 end = int(n_frames * video_percentage) for frame in trange(start, end, desc='Tracking'): cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() detections_on_frame_ = dets.get(frame, []) detections_on_frame = [] for d in detections_on_frame_: if min_width < (d.ybr - d.ytl) and min_height < (d.xbr - d.xtl): detections_on_frame.append(d) if frame == 0 or not detections_on_frame: optical_flow = None else: height, width = previous_frame.shape[:2] # get points on which to detect the flow points = [] for det in detections_on_frame: points.append([det.xtl, det.ytl]) points.append([det.xbr, det.ybr]) p0 = np.array(points, dtype=np.float32) # params for lucas-kanade optical flow lk_params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) p1, st, err = cv2.calcOpticalFlowPyrLK(previous_frame, img, p0, None, **lk_params) p0 = p0.reshape((len(detections_on_frame) * 2, 2)) p1 = p1.reshape((len(detections_on_frame) * 2, 2)) st = st.reshape(len(detections_on_frame) * 2) # flow field computed by subtracting prev points from next points flow = p1 - p0 flow[st == 0] = 0 optical_flow = np.zeros((height, width, 2), dtype=np.float32) for jj, det in enumerate(detections_on_frame): optical_flow[int(det.ytl), int(det.xtl)] = flow[2 * jj] optical_flow[int(det.ybr), int(det.xbr)] = flow[2 * jj + 1] previous_frame = img.copy() tracks, frame_tracks, max_track = update_tracks_by_overlap( tracks, detections_on_frame, max_track, refinement=False, optical_flow=optical_flow) y_true.append(gt.get(frame, [])) idf1s = [] for distance_threshold in distance_thresholds: accumulator = MOTAcumulator() y_pred = [] moving_tracks = remove_static_tracks(tracks, distance_threshold, min_track_len) detections = [] for track in moving_tracks: detections.extend(track.detections) detections = group_by_frame(detections) for frame in trange(start, end, desc='Accumulating detections'): if save_video: cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() for det in y_true[frame]: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), (0, 255, 0), 6) frame_detections = [] for det in detections.get(frame, []): frame_detections.append(det) if save_video: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), track.color, 6) cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ytl) - 15), track.color, -6) cv2.putText(img, str(det.id), (int(det.xtl), int(det.ytl)), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 6) cv2.circle(img, track.detections[-1].center, 5, track.color, -1) y_pred.append(frame_detections) if save_video: writer.append_data(cv2.resize(img, (600, 350))) accumulator.update(y_true[frame], y_pred[-1]) ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car'], sort_method='score') print(f'AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}') print('Additional metrics:') summary = accumulator.get_idf1() # As mentioned in https://github.com/cheind/py-motmetrics: # FAR = FalsePos / Frames * 100 # MOTP = (1 - MOTP) * 100 print(summary) if save_summary: with open( os.path.join( save_path, 'task1_' + sequence + '_' + camera + '_' + detector + '_' + str(distance_threshold) + '.txt'), 'w') as f: f.write(str(summary)) idf1s.append(summary['idf1']['acc'] * 100) cv2.destroyAllWindows() if save_video: writer.close() return idf1s
def task3_1(video_percentage=1): # Tracking with optical flow cap = cv2.VideoCapture('data/AICity_data/train/S03/c010/vdo.avi') n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(cap.get(cv2.CAP_PROP_FPS)) save_path = 'results/week4/task_31' os.makedirs(save_path, exist_ok=True) reader = AICityChallengeAnnotationReader( path='data/ai_challenge_s03_c010-full_annotation.xml') gt = reader.get_annotations(classes=['car']) reader = AICityChallengeAnnotationReader( path='data/AICity_data/train/S03/c010/det/det_mask_rcnn.txt') dets = reader.get_annotations(classes=['car']) if save_path: writer = imageio.get_writer(os.path.join(save_path, f'task31.gif'), fps=fps) accumulator = MOTAcumulator() y_true = [] y_pred = [] y_pred_refined = [] tracks = [] max_track = 0 previous_frame = None end = int(n_frames * video_percentage) for i, frame in tqdm(enumerate(dets.keys())): if i == end: break if save_path: cap.set(cv2.CAP_PROP_POS_FRAMES, frame) ret, img = cap.read() if i == 0: optical_flow = None else: height, width = previous_frame.shape[:2] # get points on which to detect the flow points = [] for det in detections_on_frame: points.append([det.xtl, det.ytl]) points.append([det.xbr, det.ybr]) p0 = np.array(points, dtype=np.float32) # params for lucas-kanade optical flow lk_params = dict(winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) p1, st, err = cv2.calcOpticalFlowPyrLK(previous_frame, img, p0, None, **lk_params) p0 = p0.reshape((len(detections_on_frame) * 2, 2)) p1 = p1.reshape((len(detections_on_frame) * 2, 2)) st = st.reshape(len(detections_on_frame) * 2) # flow field computed by subtracting prev points from next points flow = p1 - p0 flow[st == 0] = 0 optical_flow = np.zeros((height, width, 2), dtype=np.float32) for jj, det in enumerate(detections_on_frame): optical_flow[int(det.ytl), int(det.xtl)] = flow[2 * jj] optical_flow[int(det.ybr), int(det.xbr)] = flow[2 * jj + 1] previous_frame = img.copy() detections_on_frame = dets.get(frame, []) tracks, frame_tracks, max_track = update_tracks_by_overlap( tracks, detections_on_frame, max_track, refinement=False, optical_flow=optical_flow) frame_detections = [] for track in frame_tracks: det = track.last_detection() frame_detections.append(det) if save_path: cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ybr)), track.color, 2) cv2.rectangle(img, (int(det.xtl), int(det.ytl)), (int(det.xbr), int(det.ytl) - 15), track.color, -2) cv2.putText(img, str(det.id), (int(det.xtl), int(det.ytl)), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2) for dd in track.detections: cv2.circle(img, dd.center, 5, track.color, -1) y_pred_refined.append(frame_detections) y_pred.append(detections_on_frame) y_true.append(gt.get(frame, [])) accumulator.update(y_true[-1], y_pred_refined[-1]) if save_path: writer.append_data(cv2.resize(img, (600, 350))) cv2.destroyAllWindows() if save_path: writer.close() ap, prec, rec = mean_average_precision(y_true, y_pred, classes=['car'], sort_method='score') print(f'Original AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}') ap, prec, rec = mean_average_precision(y_true, y_pred_refined, classes=['car'], sort_method='score') print( f'After refinement AP: {ap:.4f}, Precision: {prec:.4f}, Recall: {rec:.4f}' ) print('\nAdditional metrics:') print(accumulator.get_idf1())