def set_images(self, img_list, selected_kps):
self.stitch_img = StitchedImage(img_list, target_size='max')
img_np = self.stitch_img.image
# draw reprojections
if self.draw_repr:
img_np = self.draw_reprojections(img_np, selected_kps)
# show stitch
image = QtGui.QImage(img_np.data, img_np.shape[1], img_np.shape[0],
3 * img_np.shape[1], QtGui.QImage.Format_RGB888)
image = image.scaled(img_np.shape[1], img_np.shape[0],
QtCore.Qt.IgnoreAspectRatio)
self.photo.setPixmap(QtGui.QPixmap.fromImage(image))
def _dump_vis(model, pred, gt, common,
video_list, K_list, dist_list, M_list):
from utils.plot_util import draw_skel
from utils.StitchedImage import StitchedImage
import utils.CamLib as cl
from tqdm import tqdm
# iterate frames
for i, (_, fid) in tqdm(enumerate(common), desc='Dumping Samples', total=len(common)):
# Accumulate frames
merged_list = list()
# inpaint pred/gt
for K, dist, M, v in zip(K_list, dist_list, M_list, video_list):
img = read_vid_frame(v, fid)
uv_p = cl.project(cl.trafo_coords(pred[i], M), K, dist)
img_p = draw_skel(img.copy(), model, uv_p, color_fixed='r', order='uv')
uv_gt = cl.project(cl.trafo_coords(gt[i], M), K, dist)
img_p = draw_skel(img_p, model, uv_gt, color_fixed='g', order='uv')
merged_list.append(img_p)
merged = StitchedImage(merged_list)
p = os.path.join(os.path.dirname(video_list[0]), 'eval_vis_dump/%04d.png' % i)
# cv2.imshow('img', merged.image)
# cv2.waitKey()
my_mkdir(p, is_file=True)
cv2.imwrite(p, merged.image)
model = Model(args.model)
df = build_dataflow(model, [args.set_name],
['/misc/lmbraid18/datasets/RatPose/RatTrack_paper_resub_sessions/Rat506_200306/run046_cam%d.avi'],
['/misc/lmbraid18/datasets/RatPose/RatTrack_paper_resub_sessions/Rat506_200306/pred_run046__00.json'],
is_train=False,
threaded=True, single_sample=False)
start = None
for idx, dp in enumerate(df.get_data()):
if idx >= df.size():
break
data = df2dict(dp)
img_rgb = np.round((data[data_t.image]+0.5)*255.0).astype(np.uint8)[:, :, :, ::-1]
num_cams = img_rgb.shape[0]
print('is_supervised', data[data_t.is_supervised])
img_list = list()
for i in range(num_cams):
xyz_cam = cl.trafo_coords(data[data_t.xyz_nobatch][0], data[data_t.M][i])
uv = cl.project(xyz_cam, data[data_t.K][i])
I = draw_skel(img_rgb[i], model, data[data_t.uv][i], data[data_t.vis_nobatch][0], order='uv')
img_list.append(I)
xyz = data[data_t.xyz_nobatch][0]
merge = StitchedImage(img_list, target_size=(int(0.8 * args.window_size), args.window_size))
cv2.imshow('pose labeled', merge.image[:, :, ::-1])
cv2.waitKey(0 if args.wait else 10)
boxes[:, 2] /= orig_shapes[:, 0]
boxes = np.expand_dims(boxes, 1)
scores = np.ones_like(boxes[:, :, 0])
# process boxes
pred = post_process_detections(boxes, scores,
K_list,
M_list,
imgs.shape[1:3],
verbose=False)
predictions.append(pred)
if args.show:
img_vis_list = list()
for bid in range(imgs.shape[0]):
root_uv = cl.project(cl.trafo_coords(pred['xyz'], M_list[bid]), K_list[bid])
img = cv2.circle(imgs[bid].astype(np.uint8),
(int(root_uv[0, 0]), int(root_uv[0, 1])),
radius=5,
color=(0, 255, 255),
thickness=-1)
img_vis_list.append(draw_bb(img,
pred['boxes'][bid] * imgs.shape[1],
mode='lrtb', color='g'))
merge = StitchedImage(img_vis_list)
cv2.imshow('img_bb_post', merge.image[:, :, ::-1])
cv2.waitKey(100)
json_dump(pred_file_name, predictions, verbose=True)
def run_record_cams(self):
self._setup_cams()
print(
'Start recording with hardware trigger at %.1f FPS. (press "q" to stop recording)'
% self.fps)
video_path_template = self._init_recording()
self._init_trigger()
# make cameras ready for trigger
video_writer_list = list()
for cam_name, cam in zip(self._camera_names_list, self._camera_list):
self._config_cams_hw_trigger(cam)
cam.StartGrabbing(pylon.GrabStrategy_LatestImages)
# cam.StartGrabbing(pylon.GrabStrategy_LatestImageOnly) # here you dont have any buffer
# cam.StartGrabbing(pylon.GrabStrategy_OneByOne) # here you dont get warnings if something gets skipped
video_writer_list.append(
VideoWriterFast(video_path_template % cam_name,
fps=self.fps,
codec=params_t.codec))
# start trigger
self._trigger.start()
# recording loop
k = None
last_show = 0
num_frames = defaultdict(zeros)
video_writer_q_state, cam_polling_freq, cam_last_poll = defaultdict(
zeros), defaultdict(zeros), defaultdict(zeros)
while not k == ord('q'):
try:
img_list = list()
for cid, (cam_name, cam) in enumerate(
zip(self._camera_names_list, self._camera_list)):
grabResult = cam.RetrieveResult(
params_t.cam_timeout,
pylon.TimeoutHandling_ThrowException)
if params_t.warn_frame_missing:
if grabResult.GetNumberOfSkippedImages() > 0:
print('WARNING: Missed %d frames' %
grabResult.GetNumberOfSkippedImages())
img = grabResult.GetArray()
if len(img.shape) == 2:
img = np.stack([img] * 3, -1)
if params_t.inpaint_image_id:
cv2.putText(img, '%s_%d' % (cam_name, num_frames[cid]),
(10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.6,
(0, 255, 0), 2)
# feed image to the writer
video_writer_list[cid].feed(img)
# keep track of our speed
if params_t.print_aquisition_state:
video_writer_q_state[cid] = video_writer_list[
cid].get_state()
if cam_last_poll[cid] > 0:
if cam_polling_freq[cid] > 0:
cam_polling_freq[
cid] = 0.85 * cam_polling_freq[
cid] + 0.15 * (time.time() -
cam_last_poll[cid])
else:
cam_polling_freq[cid] = time.time(
) - cam_last_poll[cid]
cam_last_poll[cid] = time.time()
num_frames[cid] += 1
# keep image around for visualization
img_list.append(img)
# show image
if (time.time() - last_show
) > params_t.show_delay and params_t.show_recorded_frames:
stitch = StitchedImage(img_list,
target_size=params_t.show_size)
cv2.imshow('cams', stitch.image)
k = cv2.waitKey(1)
last_show = time.time()
# dummy show image (otherwise 'q' key is not readable)
if not params_t.show_recorded_frames:
cv2.imshow('cams', np.ones((25, 25, 3), dtype=np.uint8))
k = cv2.waitKey(1)
# print speed measurements if needed
if params_t.print_aquisition_state:
for cid in range(len(cam_polling_freq)):
print('dev%d' % cid,
'Polling freq = %d' %
round(1.0 / cam_polling_freq[cid]),
'Queue state',
video_writer_q_state[cid],
end='\t')
print('', end='\r')
# print('', end='\n')
except genicam.TimeoutException as e:
print(e)
print('Timeout happened. Giving up.')
break
if params_t.print_aquisition_state:
print('')
# stop trigger
self._trigger.end()
del self._trigger
self._trigger = None
if self._verbosity > 2:
for cid, num in num_frames.items():
print('Device %d recorded %d frames' % (cid, num))
for cam in self._camera_list:
cam.Close()
if self._verbosity > 0:
print('Waiting for writers to finish ...')
for writer in video_writer_list:
writer.wait_to_finish()
writer.stop()
cv2.destroyAllWindows()
self._rid += 1
class FrameView(QGraphicsView):
""" The graphics view intem used to show the image tb annotated. """
def __init__(self,
parent,
width,
height,
keypoints,
kp_radius,
kp_width,
verbose=False):
super(FrameView, self).__init__(parent)
# size how large the image is in the Qt frame
self.width = width
self.height = height
self.keypoints = keypoints
self.all_names = [x['name'] for x in self.keypoints]
self.stitch_img = None
self.draw_repr = True
self.points3d = dict()
self.points3d_proj = dict()
self.frame_keypoints = dict()
self.kp_radius = kp_radius
self.kp_width = kp_width
self.verbose = verbose
self.setAcceptDrops(True) # make this class accept drags
self.reset_all_kp = False
self._zoom_scale_factor = 1.0
# create scene and fill with image
main_scene = QGraphicsScene(self)
main_scene.setBackgroundBrush(QtGui.QBrush(QtCore.Qt.black))
self.photo = QtWidgets.QGraphicsPixmapItem()
main_scene.addItem(self.photo)
self.setSceneRect(0, 0, self.width, self.height)
self.setScene(main_scene)
self.main_scene = main_scene
# settings
self.setTransformationAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
self.setResizeAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
self.setBackgroundBrush(QtGui.QBrush(QtGui.QColor(30, 30, 30)))
self.setFrameShape(QtWidgets.QFrame.NoFrame)
self.setRenderHints(QtGui.QPainter.Antialiasing
| QtGui.QPainter.SmoothPixmapTransform)
def clear(self):
self.points3d = dict()
self.points3d_proj = dict()
for kp_name in list(self.frame_keypoints.keys()):
self.delete_frame_keypoint(kp_name)
def update_frame_keypoint(self, cid, kp, pos, is_scene_pos=True):
ident = '%d_%s' % (cid, kp['name'])
if ident not in self.frame_keypoints:
if self.verbose:
print('Creating frame keypoint', ident)
# create because not existant yet
self.frame_keypoints[ident] = FrameKeypointItem(
self.photo, ident, self.kp_radius,
QtGui.QColor.fromRgb(*kp['color']), self.kp_width, kp['name'],
cid)
self.frame_keypoints[ident].kpMove.connect(self.handle_kp_moved)
self.frame_keypoints[ident].kpReset.connect(self.handle_kp_reset)
if not is_scene_pos:
# usually we get a scenepos except for when it was created by the app, then we have to do conversion
# map from original image to stitch
u, v = self.stitch_img.map_orig2stitch(pos[0], pos[1], cid)
pos = QtCore.QPoint(u, v)
# move to location
self.frame_keypoints[ident].setPos(pos)
self.frame_keypoints[ident].setVisible(True)
self.frame_keypoints[ident].is_valid = True
def handle_kp_moved(self, kp_name):
pos = self.frame_keypoints[kp_name].scenePos()
cid = self.stitch_img._get_subframe_id(pos.x(), pos.y())
if self.verbose:
print(kp_name, 'moved to subframe ', cid, ' with scene location ',
pos.x(), pos.y())
if cid == -1:
if self.verbose:
print('Special case: moved to an illegal area -> delete')
# point moved to an invalid area
# self.frame_keypoints.pop(kp_name)
self.delete_frame_keypoint(kp_name)
else:
# check if it moved into another subframe
cid_old, kp_name_stripped = self.frame_keypoints[
kp_name].cid, self.frame_keypoints[kp_name].kp_name
if cid_old != cid:
if self.verbose:
print('Special case: moved into a new area')
kp_name_new = '%d_%s' % (cid, kp_name_stripped)
if kp_name_new in self.frame_keypoints.keys():
# is this subframe was already labeled, remove the old entry
self.delete_frame_keypoint(kp_name_new)
kp = self.frame_keypoints[kp_name]
self.frame_keypoints[kp_name_new] = FrameKeypointItem(
self.photo, kp_name_new, self.kp_radius, kp.color,
self.kp_width, kp.kp_name, cid)
self.frame_keypoints[kp_name_new].kpMove.connect(
self.handle_kp_moved)
self.frame_keypoints[kp_name_new].kpReset.connect(
self.handle_kp_reset)
# move to location
self.frame_keypoints[kp_name_new].setPos(pos)
self.frame_keypoints[kp_name_new].setVisible(True)
self.frame_keypoints[kp_name_new].is_valid = True
self.delete_frame_keypoint(kp_name)
def handle_kp_reset(self, kp_name):
modifiers = QApplication.keyboardModifiers()
if self.reset_all_kp or modifiers == QtCore.Qt.ShiftModifier:
# remove all keypoints of a certain type across all views
remove_list = list()
for k, frame_kp in self.frame_keypoints.items():
if frame_kp.kp_name in kp_name:
remove_list.append(k)
for k in remove_list:
self.delete_frame_keypoint(k)
else:
self.delete_frame_keypoint(kp_name)
def delete_frame_keypoint(self, kp_name):
self.frame_keypoints[kp_name].close() # this deletes the GraphicWidget
self.frame_keypoints.pop(kp_name, None) # remove it from our dict
# remove it as child of the Pixmap item
dummyParent = QtWidgets.QGraphicsPixmapItem()
for item in self.photo.childItems():
if type(item) == FrameKeypointItem:
if item.id == kp_name:
item.setParentItem(
dummyParent
) # sets a new parent item which allows for deletion of this item
del dummyParent
def set_images(self, img_list, selected_kps):
self.stitch_img = StitchedImage(img_list, target_size='max')
img_np = self.stitch_img.image
# draw reprojections
if self.draw_repr:
img_np = self.draw_reprojections(img_np, selected_kps)
# show stitch
image = QtGui.QImage(img_np.data, img_np.shape[1], img_np.shape[0],
3 * img_np.shape[1], QtGui.QImage.Format_RGB888)
image = image.scaled(img_np.shape[1], img_np.shape[0],
QtCore.Qt.IgnoreAspectRatio)
self.photo.setPixmap(QtGui.QPixmap.fromImage(image))
def mousePressEvent(self, event: QtGui.QMouseEvent):
""" For panning the main view. """
if event.button() == QtCore.Qt.LeftButton:
self.setDragMode(QtWidgets.QGraphicsView.ScrollHandDrag)
event.accept()
super(FrameView, self).mousePressEvent(event)
def mouseReleaseEvent(self, event: QtGui.QMouseEvent):
""" For stop panning the main view. """
if event.button() == QtCore.Qt.LeftButton:
self.setDragMode(QtWidgets.QGraphicsView.NoDrag)
event.accept()
super(FrameView, self).mouseReleaseEvent(event)
def dragEnterEvent(self, event):
""" For when an example keypoint is dragged into the view. """
if event.mimeData().hasFormat('text/plain'):
event.accept()
# print('Drag enter', event.mimeData().text())
else:
event.ignore()
# print('Drag enter no data')
def dropEvent(self, event):
""" For when an example keypoint is dropped into the view. """
# print('FrameView: dropEvent')
ex_kp_name = str(event.mimeData().text())
assert ex_kp_name in self.all_names, 'Name should be defined.'
pos = self.mapToScene(event.pos())
cid = self.stitch_img._get_subframe_id(pos.x(), pos.y())
idx = self.all_names.index(ex_kp_name)
self.update_frame_keypoint(cid, self.keypoints[idx], pos)
event.accept()
super(FrameView, self).dropEvent(event)
def dragMoveEvent(self, event: QtGui.QDragMoveEvent):
# apparently one has to have this function overwise the dropevent does not work.
pass
def wheelEvent(self, event):
""" Zoom functionality."""
if not self.photo.pixmap().isNull():
# direction of scrolling
if event.angleDelta().y() > 0:
# zooming in
delta_zoom = 1.25
else:
# zooming out
delta_zoom = 0.8
# change zoom
self.updateZoomedFrame(delta_zoom)
def updateZoomedFrame(self, delta_zoom=1.0, reset=False):
""" Scales the frame view according to zoom_scale_factor. """
# reset?
if reset:
self._zoom_scale_factor = 1.0
self.fitInView()
else:
# keep track of overall zoom
self._zoom_scale_factor *= delta_zoom
# we cant make it bigger than the reference size
if self._zoom_scale_factor < 1.0:
self._zoom_scale_factor = 1.0
self.fitInView()
else:
# scale to target scale and cut the part we want
self.scale(delta_zoom, delta_zoom)
self.main_scene.setSceneRect(QtCore.QRectF())
def fitInView(self, *args):
rect = QtCore.QRectF(self.photo.pixmap().rect())
if not rect.isNull():
self.setSceneRect(QtCore.QRectF(QtCore.QPoint(0, 0), rect.size()))
unity = self.transform().mapRect(QtCore.QRectF(0, 0, 1, 1))
self.scale(1 / unity.width(), 1 / unity.height())
viewrect = self.viewport().rect()
scenerect = self.transform().mapRect(rect)
factor = min(viewrect.width() / scenerect.width(),
viewrect.height() / scenerect.height())
self.scale(factor, factor)
self.centerOn(rect.center())
def draw_reprojections(self, img_np, selected_kps):
thickness = int(round(self.kp_width))
r = int(round(self.kp_radius))
for name, pts2d in self.points3d_proj.items():
if name not in selected_kps:
continue
kp_id = self.all_names.index(name)
for cid, pt2d in enumerate(pts2d[kp_id]):
u, v, in_bounds = self.stitch_img.map_orig2stitch(
pt2d[0], pt2d[1], cid, return_bounds=True)
if in_bounds:
u, v = u.round().astype(np.int32), v.round().astype(
np.int32)
c = self.keypoints[kp_id]['color']
img_np = cv2.line(img_np, (u - r, v - r), (u + r, v + r),
color=c,
thickness=thickness)
img_np = cv2.line(img_np, (u + r, v - r), (u - r, v + r),
color=c,
thickness=thickness)
return img_np
def post_process_detections(boxes,
scores,
K_list,
M_list,
img_shape,
min_score_cand=0.2,
min_score_pick=0.1,
max_reproj_error=10.0,
verbose=True,
img=None,
logger=None):
""" Some post processing to increase the quality of bounding box detections.
We consider all bounding boxes as candidate above some min_score_cand and calculate their 2D center position.
Using all centers we aim to find a 3D hypothesis explaining as many centers as possible. Subsequently, we pick
the boxes with minimal distance to the
"""
global triang_tool
if triang_tool is None:
triang_tool = TriangTool()
output = {'boxes': None, 'xyz': None}
# calculate bounding box centers
box_centers = np.stack([
0.5 * (boxes[:, :, 1] + boxes[:, :, 3]) * img_shape[1], 0.5 *
(boxes[:, :, 0] + boxes[:, :, 2]) * img_shape[0]
], -1)
if img is not None:
# If an image was give we assume that it should be showed
img_list = list()
# show centers
for ind, I in enumerate(img):
I = I.copy()
for s, b in zip(scores[ind], box_centers[ind]):
if s < min_score_cand:
continue
c = (int(b[0]), int(b[1]))
I = cv2.circle(I, c, radius=5, color=(0, 0, 255), thickness=-1)
img_list.append(I)
from utils.StitchedImage import StitchedImage
merge = StitchedImage(img_list)
cv2.imshow('centers all', merge.image)
cv2.waitKey(10)
# resort data
points2d = list()
cams = list()
for cid in range(boxes.shape[0]):
for i in range(boxes.shape[1]):
if scores[cid, i] > min_score_cand:
points2d.append(box_centers[cid, i])
cams.append(cid)
if verbose and logger is not None:
logger.log('Cam %d contributes %d points for triangulation' %
(cid, np.sum(scores[cid] > min_score_cand)))
points2d = np.array(points2d)
if np.unique(cams).shape[0] >= 3:
# find consistent 3D hypothesis for the center of bounding boxed
point3d, inlier = triang_tool.triangulate([K_list[i] for i in cams],
[M_list[i] for i in cams],
np.expand_dims(points2d, 1),
mode=t_triangulation.RANSAC,
threshold=max_reproj_error)
if verbose and logger is not None:
logger.log('Found 3D point with %d inliers' % np.sum(inlier))
if img is not None:
img_list = list()
for ind, (I, K, M) in enumerate(zip(img, K_list, M_list)):
p2d = cl.project(cl.trafo_coords(point3d, M), K)
c = (int(p2d[0, 0]), int(p2d[0, 1]))
print(ind, c)
I = cv2.circle(I.copy(),
c,
radius=5,
color=(0, 0, 255),
thickness=-1)
img_list.append(I)
from utils.StitchedImage import StitchedImage
merge = StitchedImage(img_list)
cv2.imshow('center consistent', merge.image)
cv2.waitKey()
if np.sum(inlier) > 0:
output['xyz'] = point3d
# select optimal box wrt the center
order = [1, 3, 0, 2]
boxes_opti = list()
for cid, (K, M) in enumerate(zip(K_list, M_list)):
uv = cl.project(cl.trafo_coords(point3d, M), K)
# find bbox with minimal distance to found center
diff_l2 = np.sqrt(np.sum(np.square(box_centers[cid] - uv), -1))
diff_combined = diff_l2 / np.sqrt(
scores[cid] + 0.001
) # we want to pick something with low distance and high score
ind = np.argmin(diff_combined)
boxes_opti.append(boxes[cid, ind, order])
output['boxes'] = np.stack(boxes_opti)
return output
# If we get here its time to use the fall back solution:
# Use top scoring bbox in each frame independently
boxes_opti = list()
order = [1, 3, 0, 2]
for box, score in zip(boxes, scores):
ind = np.argmax(score)
boxes_opti.append(box[ind, order])
output['boxes'] = np.stack(boxes_opti)
if verbose and logger is not None:
logger.log(
'Using fallback solution: Best scoring box from each view, because of small amount of inliers.'
)
return output