def _fill_gaps(self, tracks):
# sort tracks by start and end timestamps
tracks = sorted(tracks, key=get_min_max_t)
# build graph where nodes are tracks and where matching tracks
# less than "track_max_gap" away are connected
graph = nx.Graph()
for i in range(len(tracks)):
graph.add_node(i)
for i, j in itertools.combinations(range(len(tracks)), 2):
# only try to match tracks with a short gap between them
ti = tracks[i][-1][0]
tj = tracks[j][0][0]
if (tj < ti) or (tj - ti > self.track_max_gap):
continue
# match tracks whose last and first position match
rectangle1 = dlib.drectangle(*tracks[i][-1][1])
rectangle2 = dlib.drectangle(*tracks[j][0][1])
if self._match(rectangle1, rectangle2):
graph.add_edge(i, j)
# merge tracks that are in the same connected component
merged_tracks = []
for group in nx.connected_components(graph):
track = [item for t in sorted(group) for item in tracks[t]]
merged_tracks.append(track)
return merged_tracks
def _fill_gaps(self, tracks):
# sort tracks by start and end timestamps
tracks = sorted(tracks, key=get_min_max_t)
# build graph where nodes are tracks and where matching tracks
# less than "track_max_gap" away are connected
graph = nx.Graph()
for i in range(len(tracks)):
graph.add_node(i)
for i, j in itertools.combinations(range(len(tracks)), 2):
# only try to match tracks with a short gap between them
ti = tracks[i][-1][0]
tj = tracks[j][0][0]
if (tj < ti) or (tj - ti > self.track_max_gap):
continue
# match tracks whose last and first position match
rectangle1 = dlib.drectangle(*tracks[i][-1][1])
rectangle2 = dlib.drectangle(*tracks[j][0][1])
if self._match(rectangle1, rectangle2):
graph.add_edge(i, j)
# merge tracks that are in the same connected component
merged_tracks = []
for group in nx.connected_components(graph):
track = [item for t in sorted(group) for item in tracks[t]]
merged_tracks.append(track)
return merged_tracks
def __init__(self, frame, bb, detect_conf):
self.t = dlib.correlation_tracker()
if not isinstance(bb, dlib.drectangle):
bb = dlib.drectangle(*bb)
self.t.start_track(frame, bb)
self.fshape = frame.shape
self.detect_conf = detect_conf
self.track_conf = None
def to_dlibFloat(self):
"""
Convert self to dlib.drectangle (float)
Returns:
dlib.drectangle:
"""
return dlib.drectangle(*self)
def _fix(self, track):
"""Fix track by merging matching forward/backward tracklets"""
fixed_track = []
for t, group in itertools.groupby(sorted(track), key=lambda x: x[0]):
group = list(group)
# make sure all positions are overlap enough
error = False
for (_, pos1, _), (_, pos2, _) in itertools.combinations(group, 2):
rectangle1 = dlib.drectangle(*pos1)
rectangle2 = dlib.drectangle(*pos2)
if self._match(rectangle1, rectangle2) == 0:
error = True
break
# status
status = "+".join(
sorted((status for _, _, status in group),
key=lambda s: {
DETECTION: 2,
FORWARD: 1,
BACKWARD: 3
}[s]))
if error:
status = "error({0})".format(status)
# average position
pos = tuple(
int(round(v))
for v in np.mean(np.vstack([p for _, p, _ in group]), axis=0))
fixed_track.append((t, pos, status))
return fixed_track
def _fix(self, track):
"""Fix track by merging matching forward/backward tracklets"""
fixed_track = []
for t, group in itertools.groupby(sorted(track), key=lambda x: x[0]):
group = list(group)
# make sure all positions are overlap enough
error = False
for (_, pos1, _), (_, pos2, _) in itertools.combinations(group, 2):
rectangle1 = dlib.drectangle(*pos1)
rectangle2 = dlib.drectangle(*pos2)
if self._match(rectangle1, rectangle2) == 0:
error = True
break
# status
status = "+".join(
sorted((status for _, _, status in group),
key=lambda s: {DETECTION: 2,
FORWARD: 1,
BACKWARD: 3}[s]))
if error:
status = "error({0})".format(status)
# average position
pos = tuple(int(round(v))
for v in np.mean(np.vstack([p for _, p, _ in group]),
axis=0))
fixed_track.append((t, pos, status))
return fixed_track
def __init__(self, pstart, pend, rois, score=None, category=None, classeme=None, vsig=None, gt_trackid=-1):
"""
bbox: drectangle
"""
# print(pstart, pend, len(rois))
assert len(rois) == pend - pstart
self.pstart = pstart
self.pend = pend
self.rois = deque(drectangle(*roi) for roi in rois)
self.score = score
self.category = category
self.classeme = classeme
# video signature
self.vsig = vsig
self.gt_trackid = gt_trackid
def _associate(self, trackers, detections):
"""Associate trackers and detections with Hungarian algorithm
Parameters
----------
trackers : dict
Dictionary where values are current trackers
and keys are trackers identifiers.
detections : list
List of detections
Returns
-------
match : dict
Dictionary where values are trackers
and keys are matched detection indices.
"""
n_trackers, n_detections = len(trackers), len(detections)
if n_trackers < 1 or n_detections < 1:
return dict()
n = max(n_trackers, n_detections)
overlap_area = np.zeros((n, n))
# list of (identifier, tracker) tuple
trackers_ = list(trackers.items())
for t, (identifier, tracker) in enumerate(trackers_):
position = tracker.get_position()
for d, detection in enumerate(detections):
rectangle = dlib.drectangle(*detection)
overlap_area[t, d] = self._match(position, rectangle)
# find the best one-to-one mapping
match = {}
mapping = self._hungarian.compute(np.max(overlap_area) - overlap_area)
for t, d in mapping:
if t >= n_trackers or d >= n_detections:
continue
if overlap_area[t, d] > 0.:
identifier, _ = trackers_[t]
match[d] = identifier
return match
def _associate(self, trackers, detections):
"""Associate trackers and detections with Hungarian algorithm
Parameters
----------
trackers : dict
Dictionary where values are current trackers
and keys are trackers identifiers.
detections : list
List of detections
Returns
-------
match : dict
Dictionary where values are trackers
and keys are matched detection indices.
"""
n_trackers, n_detections = len(trackers), len(detections)
if n_trackers < 1 or n_detections < 1:
return dict()
n = max(n_trackers, n_detections)
overlap_area = np.zeros((n, n))
# list of (identifier, tracker) tuple
trackers_ = list(trackers.items())
for t, (identifier, tracker) in enumerate(trackers_):
position = tracker.get_position()
for d, detection in enumerate(detections):
rectangle = dlib.drectangle(*detection)
overlap_area[t, d] = self._match(position, rectangle)
# find the best one-to-one mapping
match = {}
mapping = self._hungarian.compute(np.max(overlap_area) - overlap_area)
for t, d in mapping:
if t >= n_trackers or d >= n_detections:
continue
if overlap_area[t, d] > 0.:
identifier, _ = trackers_[t]
match[d] = identifier
return match
def _merge_trajs(traj_1, traj_2):
try:
assert traj_1.pend > traj_2.pstart and traj_1.pstart < traj_2.pend
except AssertionError:
print('{}-{} {}-{}'.format(traj_1.pstart, traj_1.pend, traj_2.pstart, traj_2.pend))
overlap_length = max(traj_1.pend - traj_2.pstart, 0)
for i in range(overlap_length):
roi_1 = traj_1.rois[traj_1.length() - overlap_length + i]
roi_2 = traj_2.rois[i]
left = (roi_1.left() + roi_2.left()) / 2
top = (roi_1.top() + roi_2.top()) / 2
right = (roi_1.right() + roi_2.right()) / 2
bottom = (roi_1.bottom() + roi_2.bottom()) / 2
traj_1.rois[traj_1.length() - overlap_length + i] = drectangle(left, top, right, bottom)
for i in range(overlap_length, traj_2.length()):
traj_1.predict(traj_2.rois[i])
return traj_1
def get_landmarks(self,
input_image,
all_faces=False,
detected_faces=None,
cropped_img=False,
output_csv=False,
output_csv_name='default.csv'):
with torch.no_grad():
if isinstance(input_image, str):
try:
image = io.imread(input_image)
except IOError:
print("error opening file :: ", input_image)
return None
else:
image = input_image
if cropped_img:
detected_faces = [
dlib.drectangle(0, 0, image.shape[1], image.shape[0])
]
elif detected_faces is None:
detected_faces = self.detect_faces(image)
if len(detected_faces) > 0:
landmarks = []
for i, d in enumerate(detected_faces):
if i > 0 and not all_faces:
break
if self.use_cnn_face_detector:
d = d.rect
center = torch.FloatTensor([
d.right() - (d.right() - d.left()) / 2.0,
d.bottom() - (d.bottom() - d.top()) / 2.0
])
center[1] = center[1] - (d.bottom() - d.top()) * 0.12
scale = (d.right() - d.left() + d.bottom() -
d.top()) / 195.0
inp = crop(image, center, scale)
inp = torch.from_numpy(inp.transpose(
(2, 0, 1))).float().div(255.0).unsqueeze_(0)
if self.enable_cuda:
inp = inp.cuda()
out = self.face_alignemnt_net(inp)[-1].data.cpu()
if self.flip_input:
out += flip(self.face_alignemnt_net(
flip(inp))[-1].data.cpu(),
is_label=True)
pts, pts_img = get_preds_fromhm(out, center, scale)
pts, pts_img = pts.view(68, 2) * 4, pts_img.view(68, 2)
if self.landmarks_type == LandmarksType._3D:
heatmaps = np.zeros((68, 256, 256))
for i in range(68):
if pts[i, 0] > 0:
heatmaps[i] = draw_gaussian(
heatmaps[i], pts[i], 2)
heatmaps = torch.from_numpy(heatmaps).view(
1, 68, 256, 256).float()
if self.enable_cuda:
heatmaps = heatmaps.cuda()
depth_pred = self.depth_prediciton_net(
torch.cat((inp, heatmaps),
1)).data.cpu().view(68, 1)
pts_img = torch.cat(
(pts_img, depth_pred *
(1.0 / (256.0 / (200.0 * scale)))), 1)
landmarks.append(pts_img.numpy())
else:
print("Warning: No faces were detected.")
return None
if output_csv:
with open(output_csv_name, 'w', newline='') as csvfile:
newcsv = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
temp_row = []
for i in range(landmarks.shape[0]):
temp_row.append(int(
landmarks[i, 1])) # y axis first as a convention
temp_row.append(int(landmarks[i, 0]))
newcsv.writerow(temp_row)
return landmarks
def _track(self, direction=FORWARD):
"""Actual tracking based on existing detections"""
if direction == FORWARD:
frame_cache = self._frame_cache
elif direction == BACKWARD:
frame_cache = reversed(self._frame_cache)
else:
raise NotImplementedError()
self._trackers = {}
self._confidences = {}
self._previous = {}
new_identifier = 0
for t, frame in frame_cache:
# update trackers & end those with low confidence
for identifier, tracker in list(self._trackers.items()):
confidence = tracker.update(frame)
self._confidences[identifier] = confidence
if confidence < self.track_min_confidence:
self._kill_tracker(identifier)
# match trackers with detections at time t
detections = [d for _, d, status in self._tracking_graph[t]
if status == DETECTION]
match = self._associate(self._trackers, detections)
# process all matched trackers
for d, identifier in match.items():
# connect the previous position of the tracker
# to the (current) associated detection
current = (t, detections[d], DETECTION)
self._tracking_graph.add_edge(
self._previous[identifier], current,
confidence=self._confidences[identifier])
# end the tracker
self._kill_tracker(identifier)
# process all unmatched trackers
for identifier, tracker in self._trackers.items():
# connect the previous position of the tracker
# to the current position of the tracker
position = tracker.get_position()
position = (
position.left(),
position.top(),
position.right(),
position.bottom()
)
current = (t, position, direction)
self._tracking_graph.add_edge(
self._previous[identifier], current,
confidence=self._confidences[identifier])
# save current position of the tracker for next iteration
self._previous[identifier] = current
# start new trackers for all detections
for d, detection in enumerate(detections):
# start new tracker
new_tracker = dlib.correlation_tracker()
new_tracker.start_track(frame, dlib.drectangle(*detection))
self._trackers[new_identifier] = new_tracker
# save previous (t, position, status) tuple
current = (t, detection, DETECTION)
self._previous[new_identifier] = current
# increment tracker identifier
new_identifier = new_identifier + 1
def video_object_detection(dataset,
vid,
fstart,
fend,
max_tracking_num=20,
conf_thres=0.1,
cls_nms_thres=0.3,
all_nms_thres=0.5,
vis=False):
vsig = dataset.get_video_signature(vid, fstart, fend)
print('-----Video Object Detection for {}-----'.format(vsig))
# load per-frame detection results
cls_trajs = defaultdict(list)
cnt = 0
for fid in range(fstart, fend):
with open(
os.path.join(_fr_det_root, dataset.get_frame_index(vid, fid)) +
'.pkl', 'r') as fin:
dets = pkl.load(fin)
for j, det in enumerate(dets):
if j == 0:
continue # skip background category
for i in range(det.shape[0]):
cls_trajs[j - 1].append(
Trajectory(fid - fstart,
drectangle(*det[i, :4].astype('float64')),
det[i, 4], det[i, 5:], j - 1, vsig))
cnt += 1
print('total # of per-frame detections is {}'.format(cnt))
# load frames
frames = []
for fid in range(fstart, fend):
img = io.imread(dataset.get_frame_path(vid, fid))
frames.append(img)
# track per-frame detections and nms
all_trajs = []
for j in cls_trajs.keys():
trajs = cls_trajs[j]
trajs.sort(reverse=True)
trajs = [
traj for traj in trajs[:max_tracking_num]
if traj.score > conf_thres
]
# tracking
for traj in trajs:
anchor = traj.pstart
# backward tracking
tracker = dlib.correlation_tracker()
tracker.start_track(frames[anchor], traj.roi_at(anchor))
for fid in range(anchor - 1, -1, -1):
tracker.update(frames[fid])
roi = tracker.get_position()
traj.predict(roi, reverse=True)
# forward tracking
tracker = dlib.correlation_tracker()
tracker.start_track(frames[anchor], traj.roi_at(anchor))
for fid in range(anchor + 1, len(frames)):
tracker.update(frames[fid])
roi = tracker.get_position()
traj.predict(roi)
keep = cubic_nms(trajs, cls_nms_thres)
trajs = [trajs[i] for i in keep]
print('\t# of video detections for {} is {}'. \
format(dataset.get_class_name(j), len(trajs)))
cls_trajs[j] = trajs
all_trajs.extend(trajs)
# overall nms
if len(all_trajs) > max_tracking_num:
keep = cubic_nms(all_trajs, all_nms_thres)
all_trajs = [all_trajs[i] for i in keep]
print('total # of video detections is {}'.format(len(all_trajs)))
if vis:
for traj in all_trajs:
frames = traj.draw_on(dataset, frames)
size = frames[0].shape[1], frames[0].shape[0]
save_path = dataset.get_visualization_path('vot', vid)
create_video(os.path.join(save_path, vsig), frames, 10, size, True)
return all_trajs
def to_dlibFloat(self):
return dlib.drectangle(*self)
started_tracking = False trackers = [] for i in range(num_faces): trackers.append(dlib.correlation_tracker()) for f in sorted(os.listdir(output_filename)): filename = output_filename + '/' + f frame = cv2.imread(filename) height, width, channels = frame.shape if not started_tracking: for i in range(num_faces): position = positions[i] tracker = trackers[i] detection = dlib.drectangle(position[0], position[1], position[2], position[3]) tracker.start_track(frame, detection) started_tracking = True else: for i in range(num_faces): trackers[i].update(frame) for i in range(num_faces): rect = trackers[i].get_position() try: cv2.rectangle(frame, (int(rect.left()), int(rect.top())), (int(rect.right()), int(rect.bottom())), (255, 0, 0), 2) except OverflowError: continue # do something with the aligned faces here cv2.imwrite(filename, frame)
def _track(self, direction=FORWARD):
"""Actual tracking based on existing detections"""
if direction == FORWARD:
frame_cache = self._frame_cache
elif direction == BACKWARD:
frame_cache = reversed(self._frame_cache)
else:
raise NotImplementedError()
self._trackers = {}
self._confidences = {}
self._previous = {}
new_identifier = 0
for t, frame in frame_cache:
# update trackers & end those with low confidence
for identifier, tracker in list(self._trackers.items()):
confidence = tracker.update(frame)
self._confidences[identifier] = confidence
if confidence < self.track_min_confidence:
self._kill_tracker(identifier)
# match trackers with detections at time t
detections = [
d for _, d, status in self._tracking_graph[t]
if status == DETECTION
]
match = self._associate(self._trackers, detections)
# process all matched trackers
for d, identifier in match.items():
# connect the previous position of the tracker
# to the (current) associated detection
current = (t, detections[d], DETECTION)
self._tracking_graph.add_edge(
self._previous[identifier],
current,
confidence=self._confidences[identifier])
# end the tracker
self._kill_tracker(identifier)
# process all unmatched trackers
for identifier, tracker in self._trackers.items():
# connect the previous position of the tracker
# to the current position of the tracker
position = tracker.get_position()
position = (position.left(), position.top(), position.right(),
position.bottom())
current = (t, position, direction)
self._tracking_graph.add_edge(
self._previous[identifier],
current,
confidence=self._confidences[identifier])
# save current position of the tracker for next iteration
self._previous[identifier] = current
# start new trackers for all detections
for d, detection in enumerate(detections):
# start new tracker
new_tracker = dlib.correlation_tracker()
new_tracker.start_track(frame, dlib.drectangle(*detection))
self._trackers[new_identifier] = new_tracker
# save previous (t, position, status) tuple
current = (t, detection, DETECTION)
self._previous[new_identifier] = current
# increment tracker identifier
new_identifier = new_identifier + 1
kernel=cv.getGaborKernel((ksize,ksize),sigma,theta,lamda,gamma,phi,ktype=cv.CV_32F)
plt.imshow(kernel)
plt.show
img=cv.imread('PPB/0009.jpg')
img=cv.cvtColor(img,cv.COLOR_BGR2GRAY)
detector = dlib.get_frontal_face_detector()
rects = detector(img, 1) #Detect Face
shape = predictor(img, rects)
img=dlib.drectangle(img,rec)
(x,y,w,h)=imutils.face_utils.rect_to_bb(rec)
predictor = dlib.shape_predictor(img)
cv.imshow('Original Image',img)
cv.waitKey()
cv.destroyAllWindows()
fimg=cv.filter2D(img, cv.CV_8UC3, kernel )
kernel_resize=cv.resize(kernel,(800,800))
def face_acquired():
return tracker.get_position() != dlib.drectangle(0, 0, -1, -1)
def cvbox2drectangle(bbox):
return dlib.drectangle(*(bbox[0], bbox[1], bbox[2] + bbox[0],
bbox[3] + bbox[1]))
def video_object_detection_gt(dataset, vid, fstart, fend, vis=False):
vsig = dataset.get_video_signature(vid, fstart, fend)
print('-----Video Object Detection GT for {}-----'.format(vsig))
# load gt trajectories
all_trajs = []
anno = dataset.get_annotation(vid, fstart, fend)
for trackid in anno['valid_objects']:
traj = None
for i, frame in enumerate(anno['frames']):
cnt = 0
for obj in frame['objects']:
if obj['trackid'] == trackid:
roi = drectangle(float(obj['xmin']), float(obj['ymin']),
float(obj['xmax']), float(obj['ymax']))
if traj is None:
traj = Trajectory(i, roi, 1., None,
dataset.get_class_id(obj['name']),
vsig, trackid)
else:
traj.predict(roi)
cnt += 1
assert cnt == 1, 'Object {} should appear only once in each frame.'. \
format(trackid)
assert not traj is None, 'Object {} could not be found'.format(trackid)
all_trajs.append(traj)
# compute classemes
for fid in range(fstart, fend):
with open(
os.path.join(_fr_det_root, dataset.get_frame_index(vid, fid)) +
'_norpn.pkl', 'r') as fin:
dets = pkl.load(fin)
for traj in all_trajs:
cnt = 0
det = dets[traj.category + 1]
for i in range(det.shape[0]):
roi = traj.roi_at(fid - fstart)
if tuple(det[i, :4]) == (roi.left(), roi.top(), roi.right(),
roi.bottom()):
if traj.classeme is None:
traj.classeme = det[i, 5:]
else:
traj.classeme += det[i, 5:]
cnt += 1
assert not cnt == 0, 'No object detection found at {}.'.format(fid)
assert not cnt > 1, 'Multiple object detections found at {}.'.format(
fid)
# normalize classemes
for traj in all_trajs:
traj.classeme = traj.classeme / (fend - fstart)
print('total # of video detections is {}'.format(len(all_trajs)))
if vis:
# load frames
frames = []
for fid in range(fstart, fend):
img = io.imread(dataset.get_frame_path(vid, fid))
frames.append(img)
for traj in all_trajs:
frames = traj.draw_on(dataset, frames)
size = frames[0].shape[1], frames[0].shape[0]
save_path = dataset.get_visualization_path('vot_gt', vid)
create_video(os.path.join(save_path, vsig), frames, 10, size, True)
return all_trajs