def __init__(self, debug=0): self.debug = debug self.frameCount = 0 self.timePrev = time.time() #region DEEPSORT metric = nn_matching.NearestNeighborDistanceMetric( 'cosine', matching_threshold=0.2, budget=100) self.tracker = Tracker(metric) absFilePath = os.path.abspath(__file__) fileDir = os.path.dirname(absFilePath) filePathModel = os.path.join( fileDir, 'deep_sort/resources/networks/mars-small128.pb') self.encoder = gdet.create_box_encoder(filePathModel, batch_size=1) #endregion self.detectionAndId = [] self.subscriberImageDetections = rospy.Subscriber( 'yolo_detector/output/compresseddetections', CompressedImageAndBoundingBoxes, self.callback, queue_size=1) self.publisherDetectionID = rospy.Publisher('yact/output/detectionids', DetectionAndID, queue_size=1)
def __init__(self): params = set_params() # Constructing OpenPose object allocates GPU memory opWrapper = op.WrapperPython() opWrapper.configure(params) opWrapper.start() self.opWrapper = opWrapper # Load Deep SORT model self.model_path = './deep_sort/model_data/mars-small128.pb' self.nms_max_overlap = 1.0 self.encoder = create_box_encoder(self.model_path, batch_size=1)
def __init__(self): """ Initializes DeepSORT session, i.e. object tracks and tracker internal state are persistent until object goes under garbage collector """ self.__feature_extractor = create_box_encoder( '/work/object-tracking/workspace/pretrained/mars-small128.pb') self.__max_cosine_distance = 0.9 metric = nn_matching.NearestNeighborDistanceMetric( "cosine", self.__max_cosine_distance, None) self.__tracker = Tracker(metric, n_init=5, max_age=100)
def __init__(self, config_path, image_height=480, image_width=640, image_hz=30, deepsort_modelPath="/mars_sb_14.pb", nms_conf_thresh=0.4, nms_iou_thresh=0.5, max_cosine_distance=0.6, nn_budget=200, nms_max_overlap=1.0): try: import modularmot from modularmot.utils import ConfigDecoder from deep_sort.tracker import Tracker from deep_sort import nn_matching from deep_sort.tools import generate_detections as gdet from deep_sort import preprocessing from deep_sort.detection import Detection as deep_detection import os except ImportError: raise with open(config_path) as cfg_file: config = json.load(cfg_file, cls=ConfigDecoder) project_dir = os.path.dirname(__file__) deepsort_modelPath = os.path.join(project_dir, deepsort_modelPath) self.nms_max_overlap = nms_max_overlap #Image Info self.image_height = image_height self.image_width = image_width print("Load Engine") self.mot = modularmot.MOT( [int(image_width), int(image_height)], 1.0 / int(image_hz), config['mot'], detections_only=True, draw=False, verbose=False) # deep_sort self.preprocessing = preprocessing self.deep_detection = deep_detection self.encoder = gdet.create_box_encoder(deepsort_modelPath, batch_size=1) metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) self.tracker = Tracker(metric) self.currently_busy = Event() # Base class must be called at the end due to self.service_server.spin() BaseDetectionServer.__init__(self) print("Done Init!")
def __init__(self, model_folder, max_age, max_distance=0.1, nn_budget=None, nms_max_overlap=1.0, n_init=3): # Definition of the parameters self.max_distance = max_distance self.nn_budget = nn_budget self.nms_max_overlap = nms_max_overlap # deep_sort self.encoder = gdet.create_box_encoder(model_folder, batch_size=1) #self.metric = nn_matching.NearestNeighborDistanceMetric("cosine", 0.5 , nn_budget) #max_cosine_distance self.metric = nn_matching.NearestNeighborDistanceMetric( "euclidean", max_distance, nn_budget) self.tracker = Tracker(self.metric, max_age=max_age, n_init=n_init)
def __init__(self, tracker_type="", max_cosine_distance=0.7, nn_budget=100, deep_sort_model_path=None, labels=None, nms_threshold=0.7): self.labels = labels self.nms_threshold = nms_threshold self.tracker_type = tracker_type self.nn_budget = nn_budget self.max_cosine_distance = max_cosine_distance self.trackers = [[] for i in range(len(labels))] self.encoder = None if deep_sort_model_path is not None: from deep_sort.tools import generate_detections as gen_dt self.encoder = gen_dt.create_box_encoder(deep_sort_model_path, batch_size=1)
def __init__(self, package_name, **kwargs): """ Init package attributes here :param kwargs: config params :return: """ if not os.path.isdir('model_data'): os.makedirs('model_data') # download artifacts package = dl.packages.get(package_name=package_name) if not os.path.isfile('model_data/yolo.h5'): artifact = package.project.artifacts.get(package_name=package_name, artifact_name='yolo.h5') artifact.download(local_path='model_data') if not os.path.isfile('model_data/yolo_anchors.txt'): artifact = package.project.artifacts.get( package_name=package_name, artifact_name='yolo_anchors.txt') artifact.download(local_path='model_data') if not os.path.isfile('model_data/coco_classes.txt'): artifact = package.project.artifacts.get( package_name=package_name, artifact_name='coco_classes.txt') artifact.download(local_path='model_data') if not os.path.isfile('model_data/mars-small128.pb'): artifact = package.project.artifacts.get( package_name=package_name, artifact_name='mars-small128.pb') artifact.download(local_path='model_data') ############### # load models # ############### self.yolo = YOLO() self.encoder = gdet.create_box_encoder('model_data/mars-small128.pb', batch_size=1) self.graph = tf.get_default_graph()
def detect_and_track(file_path, save_path, detection_mode="YOLO3"): # Definition of the parameters max_cosine_distance = 0.3 nn_budget = None nms_max_overlap = 1.0 # 如果要保存视频,定义视频size size = (640, 480) save_fps = 24 # use deep_sort tracker model_filename = '../deep_sort/model_data/resources/networks/mars-small128.pb' encoder = gdet.create_box_encoder(model_filename, batch_size=1) metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget) tracker = Tracker(metric) write_video_flag = True # 假设图中最多300个目标,生成300种随机颜色 colours = np.random.rand(300, 3) * 255 video_capture = cv2.VideoCapture(file_path) if write_video_flag: output_video = cv2.VideoWriter(save_path + 'output.mp4', cv2.VideoWriter_fourcc('m', 'p', '4', 'v'), save_fps, size) object_list_file = open(save_path + 'detection.txt', 'w') frame_index = -1 if detection_mode == "YOLO3": yolo = YOLO() elif detection_mode == "SSD": ssd = SSD() # appear记录每个出现过的目标存在的帧数量,number记录所有出现过的目标(不重复) appear = {} number = 0 while True: ret, frame = video_capture.read() if ret is not True: break frame = cv2.resize(frame, size) # 记录每一帧开始处理的时间 start_time = time.time() if detection_mode == "YOLO3": image = Image.fromarray(frame[..., ::-1]) # boxes为[x,y,w,h]形式坐标,detect_scores为目标分数,origin_boxes为左上角+右下角坐标形式 boxes, detect_scores, origin_boxes = yolo.detect_image(image) elif detection_mode == "SSD": rclasses, rscores, rbboxes = ssd.process_image(frame) height, width = frame.shape[0], frame.shape[1] boxes = [] # 遍历一帧图片中每个目标的(对应classes) for i in range(rclasses.shape[0]): # rbboxes原始形式为0-1范围的左上角和右下角坐标 xmin, ymin = int(rbboxes[i, 1] * width), int(rbboxes[i, 0] * height) xmax, ymax = int(rbboxes[i, 3] * width), int(rbboxes[i, 2] * height) # 转换为x,y,w,h形式的坐标 x, y, w, h = int(xmin), int(ymin), int(xmax - xmin), int(ymax - ymin) if x < 0: w = w + x x = 0 if y < 0: h = h + y y = 0 boxes.append([x, y, w, h]) boxes = np.array(boxes) features = encoder(frame, boxes) # score to 1.0 here detections = [Detection(bbox, 1.0, feature) for bbox, feature in zip(boxes, features)] # 非极大值抑制 boxes = np.array([d.tlwh for d in detections]) scores = np.array([d.confidence for d in detections]) indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores) detections = [detections[i] for i in indices] # 追踪器预测和更新 tracker.predict() tracker.update(detections) for track in tracker.tracks: if not track.is_confirmed() or track.time_since_update > 1: continue bbox = track.to_tlbr() color = (int(colours[track.track_id % 300, 0]), int(colours[track.track_id % 300, 1]), int(colours[track.track_id % 300, 2])) # (int(bbox[0]), int(bbox[1]))为左上角坐标,(int(bbox[2]), int(bbox[3]))为右下角坐标 cv2.rectangle(frame, (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])), color, 2) cv2.putText(frame, str(track.track_id), (int(bbox[0]), int(bbox[1])), 0, 5e-3 * 200, color, 2) if track.track_id in appear.keys(): appear[track.track_id] += 1 else: number += 1 appear[track.track_id] = 1 show_fps = 1. / (time.time() - start_time) cv2.putText(frame, text="FPS: " + str(int(show_fps)), org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(0, 255, 0), thickness=2) cv2.putText(frame, text="number: " + str(number), org=(3, 30), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(0, 255, 0), thickness=2) cv2.imshow('result', frame) if write_video_flag: # 保存视频每一帧 output_video.write(frame) # 更新视频帧编号 frame_index = frame_index + 1 # detection.txt写入下一帧的编号 object_list_file.write(str(frame_index) + ' ') # 写入每一帧探测到的目标的框四个点坐标 if len(boxes) != 0: for i in range(0, len(boxes)): object_list_file.write( str(boxes[i][0]) + ' ' + str(boxes[i][1]) + ' ' + str(boxes[i][2]) + ' ' + str( boxes[i][3]) + ' ') object_list_file.write('\n') # 按q可退出 if cv2.waitKey(1) & 0xFF == ord('q'): break video_capture.release() if write_video_flag: output_video.release() object_list_file.close() cv2.destroyAllWindows()
def main(args): # setting deep sort parameters tracker = Tracker(METRIC) encoder = gdet.create_box_encoder(args.tracker, batch_size=1) # Initialize the object detector detector = DetectorAPI(args.model_path) # open the video cap = cv.VideoCapture(args.video_file) # create output file if args.output_path: fourcc = cv.VideoWriter_fourcc(*'MP4V') # cap.get(3) = width, cap.get(4) = height output = cv.VideoWriter(args.output_path, fourcc, 20.0, (int(cap.get(3)), int(cap.get(4)))) # read the video frame by frame while True: check, frame = cap.read() # the end of the video? if not check: break # initialize tracking parameters sort_tracking_params = [] # do real detection # boxes are in (x_top_left, y_top_left, x_bottom_right, y_bottom_right) format boxes, scores, classes, number = detector.processFrame(frame, debug_time=True) # filter boxes due to threshold boxes = [ boxes[i] for i in range(0, number) if scores[i] > args.threshold ] # do tracking features = encoder(frame, boxes) detections = [ Detection(tbox, 1.0, feature) for tbox, feature in zip(boxes, features) ] tracker.predict() tracker.update(detections) # draw boxes due to tracking results for track in tracker.tracks: if not track.is_confirmed() or track.time_since_update > 1: continue tbox = track.to_tlwh() cv.rectangle(frame, (int(tbox[1]), int(tbox[0])), (int(tbox[3]), int(tbox[2])), (255, 255, 255), thickness=2) cv.putText(frame, 'id: {}'.format(track.track_id), (int(tbox[1]), int(tbox[0])), cv.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), thickness=2) cv.imshow(WINDOW_TITLE, frame) # save frames to specified path if args.output_path: output.write(frame) key = cv.waitKey(1) if key & 0xFF == ord('q'): break
def efficientDet_video_inference(video_src,compound_coef = 0,force_input_size=None, frame_skipping = 3, threshold=0.2,out_path=None,imshow=False, display_fps=False): #deep-sort variables # Definition of the parameters max_cosine_distance = 0.3 nn_budget = None nms_max_overlap = 1.0 model_filename = '/home/shaheryar/Desktop/Projects/Football-Monitoring/deep_sort/model_weights/mars-small128.pb' encoder = gdet.create_box_encoder(model_filename, batch_size=1) metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget) tracker = Tracker(metric,n_init=5) # efficientDet-pytorch variables iou_threshold = 0.4 use_cuda = True use_float16 = False cudnn.fastest = True cudnn.benchmark = True input_size = input_sizes[compound_coef] if force_input_size is None else force_input_size # load model model = EfficientDetBackbone(compound_coef=compound_coef, num_classes=len(obj_list)) model.load_state_dict(torch.load(f'weights/efficientdet-d{compound_coef}.pth')) model.requires_grad_(False) model.eval() if use_cuda: model = model.cuda() if use_float16: model = model.half() regressBoxes = BBoxTransform() clipBoxes = ClipBoxes() # Video capture cap = cv2.VideoCapture(video_src) frame_width = int(cap.get(3)) frame_height = int(cap.get(4)) fourcc = cv2.VideoWriter_fourcc(*'MPEG') fps = cap.get(cv2.CAP_PROP_FPS) print("Video fps",fps) if(out_path is not None): outp = cv2.VideoWriter(out_path, fourcc, fps, (frame_width, frame_height)) i=0 start= time.time() current_frame_fps=0 while True: ret, frame = cap.read() if not ret: break t1=time.time() if (frame_skipping==0 or i%frame_skipping==0): # if(True): # frame preprocessing (running detections) ori_imgs, framed_imgs, framed_metas, t1 = preprocess_video(frame, width=input_size, height=input_size) if use_cuda: x = torch.stack([fi.cuda() for fi in framed_imgs], 0) else: x = torch.stack([torch.from_numpy(fi) for fi in framed_imgs], 0) # model predict t1=time.time() with torch.no_grad(): features, regression, classification, anchors = model(x) out = postprocess(x, anchors, regression, classification, regressBoxes, clipBoxes, threshold, iou_threshold) # Post processing out = invert_affine(framed_metas, out) # decoding bbox ,object name and scores boxes,classes,scores =decode_predictions(out[0]) org_boxes = boxes.copy() t2 = time.time() - t1 # feature extraction for deep sort boxes = [convert_bbox_to_deep_sort_format(frame.shape, b) for b in boxes] features = encoder(frame,boxes) detections = [Detection(bbox, 1.0, feature) for bbox, feature in zip(boxes, features)] boxes = np.array([d.tlwh for d in detections]) # print(boxes) scores = np.array([d.confidence for d in detections]) indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores) detections = [detections[i] for i in indices] tracker.predict() tracker.update(detections) i = i + 1 img_show=frame.copy() for j in range(len(org_boxes)): img_show =drawBoxes(img_show,org_boxes[j],(255,255,0),str(tracker.tracks[j].track_id)) for track in tracker.tracks: if not track.is_confirmed() or track.time_since_update > 1: continue bbox = track.to_tlbr() x1=int(bbox[0]) y1 = int(bbox[1]) x2 = int(bbox[2]) y2=int(bbox[3]) roi= frame[y1:y2,x1:x2] cv2.rectangle(img_show, (x1, y1), (x2, y2), update_color_association(roi, track.track_id), 2) cv2.putText(img_show, str(track.track_id), (x1, y1), 0, 5e-3 * 100, (255, 255, 0), 1) if display_fps: current_frame_fps=1/t2 else: current_frame_fps=0 cv2.putText(img_show, 'FPS: {0:.2f}'.format(current_frame_fps), (30, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2, cv2.LINE_AA) if (i % int(fps) == 0): print("Processed ", str(int(i / fps)), "seconds") print("Time taken",time.time()-start) # print(color_dict) if imshow: img_show=cv2.resize(img_show,(0,0),fx=0.75,fy=0.75) cv2.imshow('Frame',img_show) # Press Q on keyboard to exit if cv2.waitKey(1) & 0xFF == ord('q'): break if out_path is not None: outp.write(img_show) cap.release() outp.release()
}) if show: cv2.namedWindow("result", cv2.WINDOW_AUTOSIZE) cv2.imshow("result", frame) if cv2.waitKey(1) & 0xFF == ord('q'): break finally: cap.release() return detection_dict if __name__ == "__main__": import sys import tensorflow as tf from keras_yolo3.yolo import YOLO from deep_sort.tools import generate_detections as gdet # load model and tracker encoder yolo = YOLO() encoder = gdet.create_box_encoder('model_data/mars-small128.pb', batch_size=1) graph = tf.get_default_graph() # run on video video_filepath = sys.argv[0] detect_video(graph=graph, yolo=yolo, encoder=encoder, video_filepath=video_filepath, mark_on_video=True, show=True)
def track(json_dir, video_dir, detect_dir, save_dir): starttime = timeit.default_timer() Path(save_dir).mkdir(parents=True, exist_ok=True) cam_datas = get_list_data(json_dir) max_cosine_distance = config['tracker']['max_cosine_distance'] nn_budget = config['tracker']['nn_budget'] nms_max_overlap = config['tracker']['nms_max_overlap'] # Deep SORT model_filename = config['tracker']['modelfile'] encoder = gdet.create_box_encoder( model_filename, batch_size=config['detector']['batchsize'] * 4) classes_map = config['detector']['classesmap'] class_names = config['detector']['classnames'] for cam_data in cam_datas: cam_name = cam_data['camName'] width = int(cam_data['imageWidth']) height = int(cam_data['imageHeight']) FPS = 10.0 video_path = os.path.join(video_dir, cam_name + '.mp4') video_cap = cv2.VideoCapture(video_path) num_frames = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT)) detect_res_path = os.path.join(detect_dir, cam_name + '.npy') bboxes = np.load(detect_res_path, allow_pickle=True) trackers = [] filtered_tracks = [] track_len_dict = [] for _ in range(len(classes_map)): metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) tracker = Tracker(metric) trackers.append(tracker) filtered_tracks.append([]) track_len_dict.append({}) for i in tqdm(range(num_frames), desc='Tracking {}'.format(cam_name)): success, frame = video_cap.read() for class_id in range(len(classes_map)): boxes, confidence, classes = map_boxes(bboxes[i], classes_map, class_id) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) features = encoder(frame, boxes) detections = [ Detection(bbox, confidence, cls, feature) for bbox, confidence, cls, feature in zip( boxes, confidence, classes, features) ] # Run non-maxima suppression. boxes = np.array([d.tlwh for d in detections]) scores = np.array([d.confidence for d in detections]) indices = preprocessing.non_max_suppression( boxes, nms_max_overlap, scores) detections = [detections[k] for k in indices] # tracking trackers[class_id].predict() trackers[class_id].update(detections) tracks = [] for track in trackers[class_id].tracks: if not track.is_confirmed() or track.time_since_update > 1: continue bbox = track.to_tlbr() tracks.append([ int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3]), class_id, track.track_id ]) if track.track_id not in track_len_dict[class_id].keys(): track_len_dict[class_id][track.track_id] = 1 else: track_len_dict[class_id][track.track_id] += 1 filtered_tracks[class_id].append(tracks) # remove short track if config['tracker']['min_len']: short_tracks = [] for class_id in range(len(classes_map)): short_track_ids = [ track_id for track_id in track_len_dict[class_id].keys() if track_len_dict[class_id][track_id] < config['tracker'] ['min_len'] ] short_tracks.append(short_track_ids) for class_id in range(len(classes_map)): for frame_id in range(num_frames): list_remove = [] for idx, track in enumerate( filtered_tracks[class_id][frame_id]): if track[-1] in short_tracks[class_id]: list_remove.append(idx) list_remove.sort(reverse=True) for idx in list_remove: filtered_tracks[class_id][frame_id].pop(idx) filtered_tracks = np.array(filtered_tracks) if save_dir: Path(save_dir).mkdir(parents=True, exist_ok=True) filepath = os.path.join(save_dir, cam_name) filtered_tracks = np.array(filtered_tracks) np.save(filepath, filtered_tracks) endtime = timeit.default_timer() print('Track time: {} seconds'.format(endtime - starttime))
args = build_argparser().parse_args() personDetector = PersonDetector(args) deepSort = False sort = True # SORT mot_tracker = Sort() # DEEP SORT PARAMS # Definition of the parameters max_cosine_distance = 0.3 nn_budget = None model_filename = 'deep_sort/models/mars-small128.pb' encoder = gdet.create_box_encoder(model_filename, batch_size=1) metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) tracker = Tracker(metric) # Open Webcam video_capturer = cv2.VideoCapture(0) video_capturer.set(cv2.CAP_PROP_FRAME_WIDTH, 720) video_capturer.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) infer_time = [] videoSize = (video_capturer.get(3), video_capturer.get(4)) while video_capturer.isOpened(): _, frame = video_capturer.read()