Exemplo n.º 1
0
    def _fill_gaps(self, tracks):

        # sort tracks by start and end timestamps
        tracks = sorted(tracks, key=get_min_max_t)

        # build graph where nodes are tracks and where matching tracks
        # less than "track_max_gap" away are connected
        graph = nx.Graph()
        for i in range(len(tracks)):
            graph.add_node(i)

        for i, j in itertools.combinations(range(len(tracks)), 2):

            # only try to match tracks with a short gap between them
            ti = tracks[i][-1][0]
            tj = tracks[j][0][0]
            if (tj < ti) or (tj - ti > self.track_max_gap):
                continue

            # match tracks whose last and first position match
            rectangle1 = dlib.drectangle(*tracks[i][-1][1])
            rectangle2 = dlib.drectangle(*tracks[j][0][1])
            if self._match(rectangle1, rectangle2):
                graph.add_edge(i, j)

        # merge tracks that are in the same connected component
        merged_tracks = []
        for group in nx.connected_components(graph):
            track = [item for t in sorted(group) for item in tracks[t]]
            merged_tracks.append(track)

        return merged_tracks
Exemplo n.º 2
0
    def _fill_gaps(self, tracks):

        # sort tracks by start and end timestamps
        tracks = sorted(tracks, key=get_min_max_t)

        # build graph where nodes are tracks and where matching tracks
        # less than "track_max_gap" away are connected
        graph = nx.Graph()
        for i in range(len(tracks)):
            graph.add_node(i)

        for i, j in itertools.combinations(range(len(tracks)), 2):

            # only try to match tracks with a short gap between them
            ti = tracks[i][-1][0]
            tj = tracks[j][0][0]
            if (tj < ti) or (tj - ti > self.track_max_gap):
                continue

            # match tracks whose last and first position match
            rectangle1 = dlib.drectangle(*tracks[i][-1][1])
            rectangle2 = dlib.drectangle(*tracks[j][0][1])
            if self._match(rectangle1, rectangle2):
                graph.add_edge(i, j)

        # merge tracks that are in the same connected component
        merged_tracks = []
        for group in nx.connected_components(graph):
            track = [item for t in sorted(group) for item in tracks[t]]
            merged_tracks.append(track)

        return merged_tracks
Exemplo n.º 3
0
 def __init__(self, frame, bb, detect_conf):
     self.t = dlib.correlation_tracker()
     if not isinstance(bb, dlib.drectangle):
         bb = dlib.drectangle(*bb)
     self.t.start_track(frame, bb)
     self.fshape = frame.shape
     self.detect_conf = detect_conf
     self.track_conf = None
Exemplo n.º 4
0
    def to_dlibFloat(self):
        """
        Convert self to dlib.drectangle (float)

        Returns:
            dlib.drectangle:

        """
        return dlib.drectangle(*self)
Exemplo n.º 5
0
    def _fix(self, track):
        """Fix track by merging matching forward/backward tracklets"""

        fixed_track = []
        for t, group in itertools.groupby(sorted(track), key=lambda x: x[0]):

            group = list(group)

            # make sure all positions are overlap enough
            error = False
            for (_, pos1, _), (_, pos2, _) in itertools.combinations(group, 2):

                rectangle1 = dlib.drectangle(*pos1)
                rectangle2 = dlib.drectangle(*pos2)

                if self._match(rectangle1, rectangle2) == 0:
                    error = True
                    break

            # status
            status = "+".join(
                sorted((status for _, _, status in group),
                       key=lambda s: {
                           DETECTION: 2,
                           FORWARD: 1,
                           BACKWARD: 3
                       }[s]))
            if error:
                status = "error({0})".format(status)

            # average position
            pos = tuple(
                int(round(v))
                for v in np.mean(np.vstack([p for _, p, _ in group]), axis=0))

            fixed_track.append((t, pos, status))

        return fixed_track
Exemplo n.º 6
0
    def _fix(self, track):
        """Fix track by merging matching forward/backward tracklets"""

        fixed_track = []
        for t, group in itertools.groupby(sorted(track), key=lambda x: x[0]):

            group = list(group)

            # make sure all positions are overlap enough
            error = False
            for (_, pos1, _), (_, pos2, _) in itertools.combinations(group, 2):

                rectangle1 = dlib.drectangle(*pos1)
                rectangle2 = dlib.drectangle(*pos2)

                if self._match(rectangle1, rectangle2) == 0:
                    error = True
                    break

            # status
            status = "+".join(
                sorted((status for _, _, status in group),
                       key=lambda s: {DETECTION: 2,
                                      FORWARD: 1,
                                      BACKWARD: 3}[s]))
            if error:
                status = "error({0})".format(status)

            # average position
            pos = tuple(int(round(v))
                        for v in np.mean(np.vstack([p for _, p, _ in group]),
                                         axis=0))

            fixed_track.append((t, pos, status))

        return fixed_track
Exemplo n.º 7
0
 def __init__(self, pstart, pend, rois, score=None, category=None, classeme=None, vsig=None, gt_trackid=-1):
     """
     bbox: drectangle
     """
     # print(pstart, pend, len(rois))
     assert len(rois) == pend - pstart
     self.pstart = pstart
     self.pend = pend
     self.rois = deque(drectangle(*roi) for roi in rois)
     self.score = score
     self.category = category
     self.classeme = classeme
     # video signature
     self.vsig = vsig
     self.gt_trackid = gt_trackid
Exemplo n.º 8
0
    def _associate(self, trackers, detections):
        """Associate trackers and detections with Hungarian algorithm

        Parameters
        ----------
        trackers : dict
            Dictionary where values are current trackers
            and keys are trackers identifiers.
        detections : list
            List of detections

        Returns
        -------
        match : dict
            Dictionary where values are trackers
            and keys are matched detection indices.
        """

        n_trackers, n_detections = len(trackers), len(detections)

        if n_trackers < 1 or n_detections < 1:
            return dict()

        n = max(n_trackers, n_detections)
        overlap_area = np.zeros((n, n))

        # list of (identifier, tracker) tuple
        trackers_ = list(trackers.items())
        for t, (identifier, tracker) in enumerate(trackers_):
            position = tracker.get_position()
            for d, detection in enumerate(detections):
                rectangle = dlib.drectangle(*detection)
                overlap_area[t, d] = self._match(position, rectangle)

        # find the best one-to-one mapping
        match = {}
        mapping = self._hungarian.compute(np.max(overlap_area) - overlap_area)
        for t, d in mapping:

            if t >= n_trackers or d >= n_detections:
                continue

            if overlap_area[t, d] > 0.:
                identifier, _ = trackers_[t]
                match[d] = identifier

        return match
Exemplo n.º 9
0
    def _associate(self, trackers, detections):
        """Associate trackers and detections with Hungarian algorithm

        Parameters
        ----------
        trackers : dict
            Dictionary where values are current trackers
            and keys are trackers identifiers.
        detections : list
            List of detections

        Returns
        -------
        match : dict
            Dictionary where values are trackers
            and keys are matched detection indices.
        """

        n_trackers, n_detections = len(trackers), len(detections)

        if n_trackers < 1 or n_detections < 1:
            return dict()

        n = max(n_trackers, n_detections)
        overlap_area = np.zeros((n, n))

        # list of (identifier, tracker) tuple
        trackers_ = list(trackers.items())
        for t, (identifier, tracker) in enumerate(trackers_):
            position = tracker.get_position()
            for d, detection in enumerate(detections):
                rectangle = dlib.drectangle(*detection)
                overlap_area[t, d] = self._match(position, rectangle)

        # find the best one-to-one mapping
        match = {}
        mapping = self._hungarian.compute(np.max(overlap_area) - overlap_area)
        for t, d in mapping:

            if t >= n_trackers or d >= n_detections:
                continue

            if overlap_area[t, d] > 0.:
                identifier, _ = trackers_[t]
                match[d] = identifier

        return match
Exemplo n.º 10
0
def _merge_trajs(traj_1, traj_2):
    try:
        assert traj_1.pend > traj_2.pstart and traj_1.pstart < traj_2.pend
    except AssertionError:
        print('{}-{} {}-{}'.format(traj_1.pstart, traj_1.pend, traj_2.pstart, traj_2.pend))
    overlap_length = max(traj_1.pend - traj_2.pstart, 0)
    for i in range(overlap_length):
        roi_1 = traj_1.rois[traj_1.length() - overlap_length + i]
        roi_2 = traj_2.rois[i]
        left = (roi_1.left() + roi_2.left()) / 2
        top = (roi_1.top() + roi_2.top()) / 2
        right = (roi_1.right() + roi_2.right()) / 2
        bottom = (roi_1.bottom() + roi_2.bottom()) / 2
        traj_1.rois[traj_1.length() - overlap_length + i] = drectangle(left, top, right, bottom)
    for i in range(overlap_length, traj_2.length()):
        traj_1.predict(traj_2.rois[i])
    return traj_1
Exemplo n.º 11
0
    def get_landmarks(self,
                      input_image,
                      all_faces=False,
                      detected_faces=None,
                      cropped_img=False,
                      output_csv=False,
                      output_csv_name='default.csv'):
        with torch.no_grad():
            if isinstance(input_image, str):
                try:
                    image = io.imread(input_image)
                except IOError:
                    print("error opening file :: ", input_image)
                    return None
            else:
                image = input_image

            if cropped_img:
                detected_faces = [
                    dlib.drectangle(0, 0, image.shape[1], image.shape[0])
                ]

            elif detected_faces is None:
                detected_faces = self.detect_faces(image)

            if len(detected_faces) > 0:
                landmarks = []
                for i, d in enumerate(detected_faces):
                    if i > 0 and not all_faces:
                        break
                    if self.use_cnn_face_detector:
                        d = d.rect

                    center = torch.FloatTensor([
                        d.right() - (d.right() - d.left()) / 2.0,
                        d.bottom() - (d.bottom() - d.top()) / 2.0
                    ])
                    center[1] = center[1] - (d.bottom() - d.top()) * 0.12
                    scale = (d.right() - d.left() + d.bottom() -
                             d.top()) / 195.0

                    inp = crop(image, center, scale)
                    inp = torch.from_numpy(inp.transpose(
                        (2, 0, 1))).float().div(255.0).unsqueeze_(0)

                    if self.enable_cuda:
                        inp = inp.cuda()

                    out = self.face_alignemnt_net(inp)[-1].data.cpu()
                    if self.flip_input:
                        out += flip(self.face_alignemnt_net(
                            flip(inp))[-1].data.cpu(),
                                    is_label=True)

                    pts, pts_img = get_preds_fromhm(out, center, scale)
                    pts, pts_img = pts.view(68, 2) * 4, pts_img.view(68, 2)

                    if self.landmarks_type == LandmarksType._3D:
                        heatmaps = np.zeros((68, 256, 256))
                        for i in range(68):
                            if pts[i, 0] > 0:
                                heatmaps[i] = draw_gaussian(
                                    heatmaps[i], pts[i], 2)
                        heatmaps = torch.from_numpy(heatmaps).view(
                            1, 68, 256, 256).float()
                        if self.enable_cuda:
                            heatmaps = heatmaps.cuda()
                        depth_pred = self.depth_prediciton_net(
                            torch.cat((inp, heatmaps),
                                      1)).data.cpu().view(68, 1)
                        pts_img = torch.cat(
                            (pts_img, depth_pred *
                             (1.0 / (256.0 / (200.0 * scale)))), 1)

                    landmarks.append(pts_img.numpy())
            else:
                print("Warning: No faces were detected.")
                return None

            if output_csv:
                with open(output_csv_name, 'w', newline='') as csvfile:
                    newcsv = csv.writer(csvfile,
                                        delimiter=',',
                                        quotechar='|',
                                        quoting=csv.QUOTE_MINIMAL)

                    temp_row = []
                    for i in range(landmarks.shape[0]):
                        temp_row.append(int(
                            landmarks[i, 1]))  # y axis first as a convention
                        temp_row.append(int(landmarks[i, 0]))

                    newcsv.writerow(temp_row)

            return landmarks
Exemplo n.º 12
0
    def _track(self, direction=FORWARD):
        """Actual tracking based on existing detections"""

        if direction == FORWARD:
            frame_cache = self._frame_cache
        elif direction == BACKWARD:
            frame_cache = reversed(self._frame_cache)
        else:
            raise NotImplementedError()

        self._trackers = {}
        self._confidences = {}
        self._previous = {}
        new_identifier = 0

        for t, frame in frame_cache:

            # update trackers & end those with low confidence
            for identifier, tracker in list(self._trackers.items()):
                confidence = tracker.update(frame)
                self._confidences[identifier] = confidence
                if confidence < self.track_min_confidence:
                    self._kill_tracker(identifier)

            # match trackers with detections at time t
            detections = [d for _, d, status in self._tracking_graph[t]
                          if status == DETECTION]
            match = self._associate(self._trackers, detections)

            # process all matched trackers
            for d, identifier in match.items():

                # connect the previous position of the tracker
                # to the (current) associated detection
                current = (t, detections[d], DETECTION)
                self._tracking_graph.add_edge(
                    self._previous[identifier], current,
                    confidence=self._confidences[identifier])

                # end the tracker
                self._kill_tracker(identifier)

            # process all unmatched trackers
            for identifier, tracker in self._trackers.items():

                # connect the previous position of the tracker
                # to the current position of the tracker
                position = tracker.get_position()
                position = (
                    position.left(),
                    position.top(),
                    position.right(),
                    position.bottom()
                )
                current = (t, position, direction)
                self._tracking_graph.add_edge(
                    self._previous[identifier], current,
                    confidence=self._confidences[identifier])

                # save current position of the tracker for next iteration
                self._previous[identifier] = current

            # start new trackers for all detections
            for d, detection in enumerate(detections):

                # start new tracker
                new_tracker = dlib.correlation_tracker()
                new_tracker.start_track(frame, dlib.drectangle(*detection))
                self._trackers[new_identifier] = new_tracker

                # save previous (t, position, status) tuple
                current = (t, detection, DETECTION)
                self._previous[new_identifier] = current

                # increment tracker identifier
                new_identifier = new_identifier + 1
Exemplo n.º 13
0
def video_object_detection(dataset,
                           vid,
                           fstart,
                           fend,
                           max_tracking_num=20,
                           conf_thres=0.1,
                           cls_nms_thres=0.3,
                           all_nms_thres=0.5,
                           vis=False):
    vsig = dataset.get_video_signature(vid, fstart, fend)
    print('-----Video Object Detection for {}-----'.format(vsig))
    # load per-frame detection results
    cls_trajs = defaultdict(list)
    cnt = 0
    for fid in range(fstart, fend):
        with open(
                os.path.join(_fr_det_root, dataset.get_frame_index(vid, fid)) +
                '.pkl', 'r') as fin:
            dets = pkl.load(fin)
        for j, det in enumerate(dets):
            if j == 0:
                continue  # skip background category
            for i in range(det.shape[0]):
                cls_trajs[j - 1].append(
                    Trajectory(fid - fstart,
                               drectangle(*det[i, :4].astype('float64')),
                               det[i, 4], det[i, 5:], j - 1, vsig))
                cnt += 1
    print('total # of per-frame detections is {}'.format(cnt))
    # load frames
    frames = []
    for fid in range(fstart, fend):
        img = io.imread(dataset.get_frame_path(vid, fid))
        frames.append(img)
    # track per-frame detections and nms
    all_trajs = []
    for j in cls_trajs.keys():
        trajs = cls_trajs[j]
        trajs.sort(reverse=True)
        trajs = [
            traj for traj in trajs[:max_tracking_num]
            if traj.score > conf_thres
        ]
        # tracking
        for traj in trajs:
            anchor = traj.pstart
            # backward tracking
            tracker = dlib.correlation_tracker()
            tracker.start_track(frames[anchor], traj.roi_at(anchor))
            for fid in range(anchor - 1, -1, -1):
                tracker.update(frames[fid])
                roi = tracker.get_position()
                traj.predict(roi, reverse=True)
            # forward tracking
            tracker = dlib.correlation_tracker()
            tracker.start_track(frames[anchor], traj.roi_at(anchor))
            for fid in range(anchor + 1, len(frames)):
                tracker.update(frames[fid])
                roi = tracker.get_position()
                traj.predict(roi)
        keep = cubic_nms(trajs, cls_nms_thres)
        trajs = [trajs[i] for i in keep]
        print('\t# of video detections for {} is {}'. \
              format(dataset.get_class_name(j), len(trajs)))
        cls_trajs[j] = trajs
        all_trajs.extend(trajs)
    # overall nms
    if len(all_trajs) > max_tracking_num:
        keep = cubic_nms(all_trajs, all_nms_thres)
        all_trajs = [all_trajs[i] for i in keep]
    print('total # of video detections is {}'.format(len(all_trajs)))
    if vis:
        for traj in all_trajs:
            frames = traj.draw_on(dataset, frames)
        size = frames[0].shape[1], frames[0].shape[0]
        save_path = dataset.get_visualization_path('vot', vid)
        create_video(os.path.join(save_path, vsig), frames, 10, size, True)
    return all_trajs
Exemplo n.º 14
0
 def to_dlibFloat(self):
     return dlib.drectangle(*self)
started_tracking = False

trackers = []
for i in range(num_faces):
	trackers.append(dlib.correlation_tracker())

for f in sorted(os.listdir(output_filename)):
	filename = output_filename + '/' + f
	frame = cv2.imread(filename)
	height, width, channels = frame.shape

   	if not started_tracking:
		for i in range(num_faces):
			position = positions[i]
			tracker = trackers[i]
			detection = dlib.drectangle(position[0], position[1], position[2], position[3])
			tracker.start_track(frame, detection)
		started_tracking = True
   	else:
		for i in range(num_faces):
			trackers[i].update(frame)

	for i in range(num_faces):
		rect = trackers[i].get_position()
		try:
			cv2.rectangle(frame, (int(rect.left()), int(rect.top())), (int(rect.right()), int(rect.bottom())), (255, 0, 0), 2)
		except OverflowError:
			continue

	# do something with the aligned faces here
	cv2.imwrite(filename, frame)
Exemplo n.º 16
0
    def _track(self, direction=FORWARD):
        """Actual tracking based on existing detections"""

        if direction == FORWARD:
            frame_cache = self._frame_cache
        elif direction == BACKWARD:
            frame_cache = reversed(self._frame_cache)
        else:
            raise NotImplementedError()

        self._trackers = {}
        self._confidences = {}
        self._previous = {}
        new_identifier = 0

        for t, frame in frame_cache:

            # update trackers & end those with low confidence
            for identifier, tracker in list(self._trackers.items()):
                confidence = tracker.update(frame)
                self._confidences[identifier] = confidence
                if confidence < self.track_min_confidence:
                    self._kill_tracker(identifier)

            # match trackers with detections at time t
            detections = [
                d for _, d, status in self._tracking_graph[t]
                if status == DETECTION
            ]
            match = self._associate(self._trackers, detections)

            # process all matched trackers
            for d, identifier in match.items():

                # connect the previous position of the tracker
                # to the (current) associated detection
                current = (t, detections[d], DETECTION)
                self._tracking_graph.add_edge(
                    self._previous[identifier],
                    current,
                    confidence=self._confidences[identifier])

                # end the tracker
                self._kill_tracker(identifier)

            # process all unmatched trackers
            for identifier, tracker in self._trackers.items():

                # connect the previous position of the tracker
                # to the current position of the tracker
                position = tracker.get_position()
                position = (position.left(), position.top(), position.right(),
                            position.bottom())
                current = (t, position, direction)
                self._tracking_graph.add_edge(
                    self._previous[identifier],
                    current,
                    confidence=self._confidences[identifier])

                # save current position of the tracker for next iteration
                self._previous[identifier] = current

            # start new trackers for all detections
            for d, detection in enumerate(detections):

                # start new tracker
                new_tracker = dlib.correlation_tracker()
                new_tracker.start_track(frame, dlib.drectangle(*detection))
                self._trackers[new_identifier] = new_tracker

                # save previous (t, position, status) tuple
                current = (t, detection, DETECTION)
                self._previous[new_identifier] = current

                # increment tracker identifier
                new_identifier = new_identifier + 1
Exemplo n.º 17
0
kernel=cv.getGaborKernel((ksize,ksize),sigma,theta,lamda,gamma,phi,ktype=cv.CV_32F)
plt.imshow(kernel)
plt.show


img=cv.imread('PPB/0009.jpg')
img=cv.cvtColor(img,cv.COLOR_BGR2GRAY)
    
detector = dlib.get_frontal_face_detector()
rects = detector(img, 1)    #Detect Face

shape = predictor(img, rects)


img=dlib.drectangle(img,rec)


(x,y,w,h)=imutils.face_utils.rect_to_bb(rec)
 
predictor = dlib.shape_predictor(img)


cv.imshow('Original Image',img)
cv.waitKey()
cv.destroyAllWindows()



fimg=cv.filter2D(img, cv.CV_8UC3, kernel )
kernel_resize=cv.resize(kernel,(800,800))
Exemplo n.º 18
0
 def face_acquired():
     return tracker.get_position() != dlib.drectangle(0, 0, -1, -1)
Exemplo n.º 19
0
def cvbox2drectangle(bbox):
    return dlib.drectangle(*(bbox[0], bbox[1], bbox[2] + bbox[0],
                             bbox[3] + bbox[1]))
Exemplo n.º 20
0
def video_object_detection_gt(dataset, vid, fstart, fend, vis=False):
    vsig = dataset.get_video_signature(vid, fstart, fend)
    print('-----Video Object Detection GT for {}-----'.format(vsig))
    # load gt trajectories
    all_trajs = []
    anno = dataset.get_annotation(vid, fstart, fend)
    for trackid in anno['valid_objects']:
        traj = None
        for i, frame in enumerate(anno['frames']):
            cnt = 0
            for obj in frame['objects']:
                if obj['trackid'] == trackid:
                    roi = drectangle(float(obj['xmin']), float(obj['ymin']),
                                     float(obj['xmax']), float(obj['ymax']))
                    if traj is None:
                        traj = Trajectory(i, roi, 1., None,
                                          dataset.get_class_id(obj['name']),
                                          vsig, trackid)
                    else:
                        traj.predict(roi)
                    cnt += 1
            assert cnt == 1, 'Object {} should appear only once in each frame.'. \
                format(trackid)
        assert not traj is None, 'Object {} could not be found'.format(trackid)
        all_trajs.append(traj)
    # compute classemes
    for fid in range(fstart, fend):
        with open(
                os.path.join(_fr_det_root, dataset.get_frame_index(vid, fid)) +
                '_norpn.pkl', 'r') as fin:
            dets = pkl.load(fin)
        for traj in all_trajs:
            cnt = 0
            det = dets[traj.category + 1]
            for i in range(det.shape[0]):
                roi = traj.roi_at(fid - fstart)
                if tuple(det[i, :4]) == (roi.left(), roi.top(), roi.right(),
                                         roi.bottom()):
                    if traj.classeme is None:
                        traj.classeme = det[i, 5:]
                    else:
                        traj.classeme += det[i, 5:]
                    cnt += 1
            assert not cnt == 0, 'No object detection found at {}.'.format(fid)
            assert not cnt > 1, 'Multiple object detections found at {}.'.format(
                fid)
    # normalize classemes
    for traj in all_trajs:
        traj.classeme = traj.classeme / (fend - fstart)
    print('total # of video detections is {}'.format(len(all_trajs)))
    if vis:
        # load frames
        frames = []
        for fid in range(fstart, fend):
            img = io.imread(dataset.get_frame_path(vid, fid))
            frames.append(img)
        for traj in all_trajs:
            frames = traj.draw_on(dataset, frames)
        size = frames[0].shape[1], frames[0].shape[0]
        save_path = dataset.get_visualization_path('vot_gt', vid)
        create_video(os.path.join(save_path, vsig), frames, 10, size, True)
    return all_trajs