def test_mr_sd(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
ma.analyse((0, 1000), 5)
ma.analyse((500, 1000), 5)
ma.analyse((0, 0), 0)
ma.analyse((0, 216), 0)
print(drawing.view_corner)
assert drawing.view_corner == (670, 1080)
def test_analyse_gesture_same_in_motion(self):
# Setup
width = 1920
height = 1080
pos_1 = 0, 0
pos_2 = 1, 6
pos_3 = 5, 4
pos_4 = 2, 4
pos_5 = 11, 0
gesture = 1
drawing = Drawing(width, height)
ma = MotionAnalyser(width, height, drawing)
ma.analyse(pos_1, gesture)
ma.analyse(pos_2, gesture)
ma.analyse(pos_3, gesture)
ma.analyse(pos_4, gesture)
ma.analyse(pos_5, gesture)
exp_gesture = gesture
exp_pos = pos_5
exp_total_dx = 11
exp_total_dy = 0
assert exp_gesture == ma.ex_gesture
assert exp_pos == pytest.approx(ma.ex_pos)
assert round(abs(exp_total_dx - ma.total_dx), 7) == 0
assert round(abs(exp_total_dy - ma.total_dy), 7) == 0
def yolo_detection(gesture_lock, phone_cam):
labels = open('../model/_darknet.labels').read().strip().split('\n')
cfg, weights = '../model/custom-yolov4-detector.cfg', '../model/yolov4-hand-gesture.weights'
net = cv2.dnn.readNetFromDarknet(cfgFile=cfg, darknetModel=weights)
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
layer_names = net.getLayerNames()
layer_names = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]
cap, fps = init_cam(phone_cam)
# W = cap.get(3)
# H = cap.get(4)
W = 1920
H = 1080
ma = MotionAnalyser(W, H)
while True:
frame = cap.read()
frame = cv2.flip(frame, 1)
class_id = make_prediction(net, layer_names, frame, conf_threshold=0.9)
if class_id is not None: # keep previous state, if None
gesture_lock.set_gesture(gesture=labels[class_id])
fps.update()
if keyboard.is_pressed('esc'): # can't use cv's waitKey, cuz no window
break
fps.stop()
print("Mean fps for detection:", round(fps.fps(), 2))
def test_analyse_gesture_change_on_start(self):
# Setup
width = 1920
height = 1080
pos = 0, 0
gesture = 1
drawing = Drawing(width, height)
ma = MotionAnalyser(width, height, drawing)
ma.analyse(pos, gesture)
exp_gesture = gesture
exp_pos = pos
exp_total_dx = 0
exp_total_dy = 0
assert exp_gesture == ma.ex_gesture
assert exp_pos == pytest.approx(ma.ex_pos)
assert round(abs(exp_total_dx - ma.total_dx), 7) == 0
assert round(abs(exp_total_dy - ma.total_dy), 7) == 0
def test_analyse_gesture_none(self):
# Setup
width = 1920
height = 1080
pos = 1.23, 3.55
gesture = None
drawing = Drawing(width, height)
ma = MotionAnalyser(width, height, drawing)
ma.analyse(pos, gesture)
ma.analyse(pos, gesture)
exp_gesture = gesture
exp_pos = 0, 0
exp_total_dx = 0
exp_total_dy = 0
assert exp_gesture == ma.ex_gesture
assert exp_pos == pytest.approx(ma.ex_pos)
assert round(abs(exp_total_dx - ma.total_dx), 7) == 0
assert round(abs(exp_total_dy - ma.total_dy), 7) == 0
def test_draw_after_moving_left(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
frame = np.zeros((1080, 1920, 3), dtype="uint8")
ma.analyse((0, 1000), 5)
drawing.process_frame(frame=frame,
x=None,
y=None,
area=None,
action="Erasing")
ma.analyse((500, 1000), 5)
drawing.process_frame(frame=frame,
x=None,
y=None,
area=None,
action="Erasing")
drawing.process_frame(frame=frame,
x=300,
y=300,
area=200,
action="Blue")
drawing.process_frame(frame=frame,
x=300,
y=300,
area=200,
action="Blue")
drawing.process_frame(frame=frame,
x=400,
y=400,
area=200,
action="Blue")
print(drawing.view_corner)
for x in range(300, 400):
assert (drawing.canvas[x + 1080, x + 670] == np.array([255, 0,
0])).all()
def test_draw_after_color_erase_drawn(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
ma.analyse((0, 0), 5)
ma.analyse((0, 300), 5)
frame = np.zeros((1080, 1920, 3), dtype="uint8")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Erasing")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Erasing")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Erasing")
for x in range(300, 400):
assert (drawing.canvas[x + 330, x + 1920] == np.array([0, 0, 0])).all()
def test_draw_after_color_green_to_blue(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
ma.analyse((0, 0), 5)
ma.analyse((0, 300), 5)
frame = np.zeros((1080, 1920, 3), dtype="uint8")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Blue")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Blue")
drawing.process_frame(frame=frame, x=350, y=350, area=196, action="Blue")
for x in range(300, int(350 - 14 / math.sqrt(2))):
assert (drawing.canvas[x + 330, x + 1920] == np.array([0, 255, 0])).all()
for x in range(int(350 - 14 / math.sqrt(2)) + 1, 400):
assert (drawing.canvas[x + 330, x + 1920] == np.array([255, 0, 0])).all()
def test_analyse_gesture_change(self):
# Setup
width = 1920
height = 1080
pos = 0, 0
gesture_1 = 1
gesture_2 = 2
gesture_3 = 3
drawing = Drawing(width, height)
ma = MotionAnalyser(width, height, drawing)
ma.analyse(pos, gesture_1)
ma.analyse(pos, gesture_1)
ma.analyse(pos, gesture_2)
ma.analyse(pos, gesture_2)
ma.analyse(pos, gesture_1)
ma.analyse(pos, gesture_3)
exp_gesture = gesture_3
exp_pos = pos
exp_total_dx = pos[0]
exp_total_dy = pos[0]
assert exp_gesture == ma.ex_gesture
assert exp_pos == pytest.approx(ma.ex_pos)
assert round(abs(exp_total_dx - ma.total_dx), 7) == 0
assert round(abs(exp_total_dy - ma.total_dy), 7) == 0
def test_draw_after_scaling_down(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
frame = np.zeros((1080, 1920, 3), dtype="uint8")
ma.analyse((0, 216), 0)
drawing.process_frame(frame=frame,
x=None,
y=None,
area=None,
action="Green")
ma.analyse((0, 0), 0)
drawing.process_frame(frame=frame,
x=None,
y=None,
area=None,
action="Green")
print(drawing.view_corner, drawing.scale_factor)
drawing.process_frame(frame=frame,
x=300,
y=300,
area=200,
action="Blue")
drawing.process_frame(frame=frame,
x=300,
y=300,
area=200,
action="Blue")
drawing.process_frame(frame=frame,
x=400,
y=400,
area=200,
action="Blue")
print(np.where(drawing.canvas > 0))
for x in range(300, 400):
assert (drawing.canvas[int(x * 0.5 + 1080),
int(x * 0.5 + 1920)] == np.array(
[255, 0, 0])).all()
def test_draw_after_color_green_then_blue_and_cross_black(self):
drawing = Drawing(1920, 1080)
ma = MotionAnalyser(1080, 1920, drawing)
ma.analyse((0, 0), 5)
ma.analyse((0, 300), 5)
frame = np.zeros((1080, 1920, 3), dtype="uint8")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=300, y=300, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Green")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Blue")
drawing.process_frame(frame=frame, x=400, y=400, area=196, action="Blue")
drawing.process_frame(frame=frame, x=350, y=350, area=196, action="Blue")
drawing.process_frame(frame=frame, x=400, y=300, area=196, action="Black")
drawing.process_frame(frame=frame, x=400, y=300, area=196, action="Black")
drawing.process_frame(frame=frame, x=300, y=400, area=196, action="Black")
# cross top-left part
for x in range(300, int(350 - 14 / math.sqrt(2))):
assert (drawing.canvas[x + 330, x + 1920] == np.array([0, 255, 0])).all(), x
# cross intersection part
for x in range(int(350 - 14 / math.sqrt(2)) + 1, int(350 + 14 / math.sqrt(2))):
assert (drawing.canvas[x + 330, x + 1920] == np.array([255, 255, 255])).all(), x
# after cross intersection, must be same as topleft
for x in range(int(350 + 14 / math.sqrt(2)) + 2, 400):
assert (drawing.canvas[x + 330, x + 1920] == np.array([255, 0, 0])).all(), x
def main_cam(phone_cam,
video_path=None,
imgs_paths=None,
default_pen_color=None):
cap, fps = init_cam(phone_cam, video_path, imgs_paths)
fps_count = 0.0
W = cap.get(3)
H = cap.get(4)
drawing = Drawing(W, H, default_pen_color)
ma = MotionAnalyser(W, H, drawing)
logger = Logger(drawing, ma)
mp_hands = mp.solutions.hands
detector = mp_hands.Hands(min_detection_confidence=0.7,
min_tracking_confidence=0.5)
pairs = {
3: "Yellow",
5: "Erasing",
0: "Green",
2: "Brown",
1: "Blue",
4: "Black"
}
cv2.namedWindow("Cam", cv2.WINDOW_NORMAL)
cv2.setWindowProperty("Cam", cv2.WND_PROP_FULLSCREEN, 1)
n_skipped_frames = 0
while True:
frame = cap.read()
if isinstance(frame, tuple):
frame = frame[1]
if frame is None or frame.size == 0:
n_skipped_frames += 1
print("Skipped", n_skipped_frames)
if n_skipped_frames == 90:
print("End of sequence")
break
continue
frame = cv2.flip(frame, 1)
cv2.setMouseCallback('Cam', callback, (drawing, frame))
n_fingers_l, center_l = count_fingers(frame, detector)
x, y, area = drawing.find_pen(frame)
if x is None: # no pen in frame
ma.analyse(center_l, n_fingers_l)
if n_fingers_l is not None:
action = pairs[n_fingers_l]
text = action + " (" + str(n_fingers_l) + ") "
else:
action = None
text = "None "
frame = drawing.process_frame(frame, x, y, area, action)
cv2.putText(frame,
text=text + str(x),
org=(30, 100),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=2,
color=(0, 255, 20),
thickness=3)
fps.update()
if fps._numFrames == 25:
fps.stop()
fps_count = fps.fps()
fps = FPS().start()
cv2.putText(frame,
text=str(round(fps_count, 1)),
org=(1750, 50),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=2,
color=(0, 255, 20),
thickness=3)
cv2.putText(frame,
text=str(round(drawing.scale_factor, 2)),
org=(1750, 1000),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=2,
color=(0, 255, 20),
thickness=3)
cv2.imshow('Cam', frame)
logger.log(n_fingers_l, center_l, (x, y), area)
if cv2.waitKey(1) & 0xFF == 27:
break
cv2.destroyAllWindows()
return logger
net = cv2.dnn.readNetFromDarknet(cfgFile=cfg, darknetModel=weights)
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
layer_names = net.getLayerNames()
layer_names = [
layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()
]
cap = cv2.VideoCapture(0)
address = "http://192.168.1.193:8080/video"
cap.open(address)
fps = FPS().start()
W = cap.get(3)
H = cap.get(4)
ma = MotionAnalyser(W, H)
while cap.isOpened():
ret, frame = cap.read()
if not ret:
print('Video file finished.')
break
frame = cv2.flip(frame, 1)
conf, class_id, center = make_prediction(net,
layer_names,
frame,
conf_threshold=0.6)
if class_id is None:
text = "None"
else:
