def __init__(self):
rospy.init_node('rgb_object_detection')
self.fps_reduce = 0
self.palm_model_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/models/palm_detection.tflite"
self.landmark_model_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/models/hand_landmark.tflite"
self.anchors_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/data/anchors.csv"
self.estimator_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/logregest_1200.pickle"
self.bridge = CvBridge()
self.estimator = self.get_estimator()
self.detector = HandTracker(self.palm_model_path,
self.landmark_model_path,
self.anchors_path,
box_shift=0.2,
box_enlarge=1.3)
self.start_f = False
self.under_line = False
self.end_flag = False
self.count_moves = 0
self.delay_counter = 0
self.end_frame = 0
self.img_sub = rospy.Subscriber("/camera/color/image_raw",
Image,
self.img_to_cv2,
queue_size=1)
self.res_pub = rospy.Publisher("/RPS_node/result",
String,
queue_size=1)
self.res_pub.publish("playing a b")
def __init__(self):
threading.Thread.__init__(self)
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0,
box_enlarge=1.5)
def __init__(self, palm_recog_path, kp_recog_path, anchor_path):
self.detector = HandTracker(palm_recog_path,
kp_recog_path,
anchor_path,
box_shift=0.2,
box_enlarge=1.3)
self.finger_bend = [False] * 5
def __init__(self):
# Initializing the HandTracker #################################################
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.1,
box_enlarge=1.5)
self.kp_single_int = tuple([0, 0]) # initialized as the result
# Setting keypoints pairs ###############################
# self.pair_WRONG = [[1, 2], [2, 3], [3, 4],
# [5, 6], [6, 7], [7, 8],
# [9, 10], [10, 11], [11, 12],
# [13, 14], [14, 15], [15, 16],
# [17, 18], [18, 19],
# [19, 20]] # Adding a WRONG at the end of function name just means this is a
# # wrong way of calculation
self.pair = [[0, 1, 2], [1, 2, 3], [2, 3, 4], [0, 5, 6], [5, 6, 7],
[6, 7, 8], [0, 9, 10], [9, 10, 11], [10, 11, 12],
[0, 13, 14], [13, 14, 15], [14, 15, 16], [0, 17, 18],
[17, 18, 19], [18, 19, 20]]
self.stem_pair = [0, 9] # palm points and root middle-finger point
def __init__(self):
# Initializing the HandTracker #################################################
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.1,
box_enlarge=1.5)
self.kp_single_int = tuple([0, 0]) # initialized as the result
# Setting keypoints pairs ###############################
self.pair = [[1, 2], [2, 3], [3, 4], [5, 6], [6, 7], [7, 8], [9, 10],
[10, 11], [11, 12], [13, 14], [14, 15], [15, 16],
[17, 18], [18, 19], [19, 20]]
self.stem_pair = [0, 9] # palm points and root middle-finger point
def main(args):
det = HandTracker('models/palm_detection_without_custom_op.tflite',
'models/hand_landmark_3d.tflite',
'data/anchors.csv',
box_shift=-0.5,
box_enlarge=2.6)
video_name = args.video
out_dir = args.outdir
print('Processing {}'.format(video_name))
for frame_i, img in enumerate(read_video(video_name)):
img_rgb = img[:, :, ::-1]
kpts, boxs = det(img_rgb)
out_file = out_dir + '/' + str(frame_i) + '.jpg'
if boxs is None:
out_img = img
else:
out_img = draw_hands(img, kpts, boxs)
cv2.imwrite(out_file, out_img)
def __init__(self, numGestures, numFramesPerGesture,
minDescriptorsPerFrame, numWords, descType, kernel, numIter,
parent):
self.numGestures = numGestures
self.numFramesPerGesture = numFramesPerGesture
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.desList = []
self.voc = None
self.classifier = None
self.windowName = "Training preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.kernel = kernel
self.numIter = numIter
self.numDefects = None
self.firstFrameList = []
self.trainLabels = []
def __init__(self, numGestures, minDescriptorsPerFrame, numWords, descType,
numPredictions, parent):
self.numGestures = numGestures
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.classifier = None
self.windowName = "Testing preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.predictionList = [-1] * numPredictions
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.numSideFrames = 10
self.prevFrameList = np.zeros(
(self.numSideFrames, self.parent.imHeight / self.numSideFrames,
self.parent.imWidth / self.numSideFrames, 3), "uint8")
self.numPrevFrames = 0
self.predictionScoreThreshold = 0.2
self.learningRate = 0.01
self.numReinforce = 1
# 4 | | | |
# | 6 10 14 18
# 3 | | | |
# | 5---9---13--17
# 2 \ /
# \ \ /
# 1 \ /
# \ \ /
# ------0-
connections = [(0, 1), (1, 2), (2, 3), (3, 4), (5, 6), (6, 7), (7, 8), (9, 10),
(10, 11), (11, 12), (13, 14), (14, 15), (15, 16), (17, 18),
(18, 19), (19, 20), (0, 5), (5, 9), (9, 13), (13, 17), (0, 17)]
detector = HandTracker(PALM_MODEL_PATH,
LANDMARK_MODEL_PATH,
ANCHORS_PATH,
box_shift=0.2,
box_enlarge=1.3)
def runModel(img, threshold):
image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (256, 256))
points = detector(image, threshold)
output_json = list()
for points_ in points:
if points_ is not None:
#save outputs into json
model_output_json = {}
def start(self):
self.tracker = HandTracker(self.classifiers, debug=True)
self.tracker.start()
import sys
import numpy as np
sys.path.append("/api")
from pose_util import hand_from_raw
sys.path.append("/hand_tracking")
from hand_tracker import HandTracker
palm_model_path = "/hand_tracking/models/palm_detection.tflite"
landmark_model_path = "/hand_tracking/models/hand_landmark_3d.tflite"
anchors_path = "/hand_tracking/data/anchors.csv"
detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.2,
box_enlarge=1.3)
reader = imageio.get_reader('/video.mp4')
counter = 0
for i, img in enumerate(reader):
counter += 1
if img.shape[0] == 256 and img.shape[1] == 256:
print("Skipping hand detection")
hands = [{"bbox": np.array([(0, 256), (256, 256), (256, 0), (0, 0)])}]
hands = [detector.get_landmarks(img, h) for h in hands]
else:
hands = detector(img)
from matplotlib.patches import Polygon
from hand_tracker import HandTracker
from scipy.signal import savgol_filter
import cv2
import numpy as np
import pickle
import sys
import glob
import os
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
detector = HandTracker(palm_model_path, landmark_model_path, anchors_path)
def write_kps(infile, outfile):
cap = cv2.VideoCapture(infile)
data = []
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_counter = 0
while True:
res, frame = cap.read()
if not res:
if frame_counter == total_frames:
print('finished, writing output to', outfile)
pickle.dump(data, open(outfile, 'wb'))
else:
def __init__(self, options):
self.options = options
self.hand_tracker = HandTracker(self.options)
self.filter = Filter2D()
self.hand_prev = None
self.idle_frames = 0
import numpy as np
import cv2
from hand_tracker import HandTracker
det = HandTracker('models/palm_detection_without_custom_op.tflite',
'models/hand_landmark_3d.tflite',
'data/anchors.csv',
box_shift=-0.5,
box_enlarge=2.6)
in_bgr = cv2.imread('data/test_img1.jpg')
in_rgb = in_bgr[:, :, ::-1]
list_keypoints, list_bbox = det(in_rgb)
out_img = np.copy(in_bgr)
# point size
ps = int(np.ceil(min(out_img.shape[0], out_img.shape[1]) / 256))
if list_keypoints is not None:
for idx in range(len(list_keypoints)):
keypoints = list_keypoints[idx]
bbox = list_bbox[idx]
for i in range(4):
j = (i + 1) % 4
p0 = (int(bbox[i, 0] + 0.5), int(bbox[i, 1] + 0.5))
p1 = (int(bbox[j, 0] + 0.5), int(bbox[j, 1] + 0.5))
cv2.line(out_img, p0, p1, (0, 0, 255), ps)
for i in range(keypoints.shape[0]):
p = (int(keypoints[i, 0] + 0.5), int(keypoints[i, 1] + 0.5))
cv2.circle(out_img, p, ps, (255, 0, 0), ps)
cv2.imwrite('out.jpg', out_img)
def learningFrames(vc):
t = 0
while(vc.isOpened()):
ret,im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
handTracker.colorProfiler.draw_color_windows(im, imhsv)
cv2.imshow("color", im)
k = cv2.waitKey(1)
if k == 32: # space
break
elif k == 27:
sys.exit(0)
handTracker.colorProfiler.run()
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
imhsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
binaryIM = handTracker.get_binary_image(imhsv)
cnt,hull,centroids, defects = handTracker.initialize_contour(binaryIM)
while True:
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
imhsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
imgray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
binaryIm = handTracker.get_binary_image(imhsv)
cnt,hull,centroids, defects = handTracker.get_contour(binaryIm)
frameCopy = 1*frame
testData = None
prediction = -1
score = -1
if cnt is not None:
numDefects = defects.shape[0]
cropImage,cropPoints = handTracker.get_cropped_image_from_cnt(frame, cnt, 0.05)
cropImageGray = handTracker.get_cropped_image_from_points(imgray, cropPoints)
kp = featureExtractor.get_keypoints(cropImageGray)
cropCnt = handTracker.get_cropped_contour(cnt, cropPoints)
kp = featureExtractor.get_keypoints_in_contour(kp, cropCnt)
kp, des = featureExtractor.compute_descriptors(cropImageGray, kp)
if des is not None and des.shape[0] >= 0:
featureExtractor.draw_keypoints(cropImage, kp)
des = np.float32(des)
if des is not None and des.shape[0] >= 10 and is_hand(defects):
voc, variance = kmeans(des, numWords, 30)
words, distance = vq(des, voc)
testData = np.zeros(numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(numWords)
testData = np.divide(testData, normTestData)
states.append([testData, np.zeros(numWords), testData])
class_probabilities = clf.predict_proba(testData)
print t
print class_probabilities
if max(class_probabilities) > 0.3:
cv2.imshow("frames", cropImage)
t = t + 1
else:
handTracker.draw_on_image(frameCopy, cnt=False, hullColor=(0,0,255))
else:
prediction = -1
cv2.imshow('testing', frameCopy)
def no_hand(vc):
firstFrame = None
hand = False
while True:
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
text = "Unoccupied"
hand = False
frame = imutils.resize(frame, width=500)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (21, 21), 0)
if firstFrame is None:
firstFrame = gray
continue
frameDelta = cv2.absdiff(firstFrame, gray)
thresh = cv2.threshold(frameDelta, 25, 255, cv2.THRESH_BINARY)[1]
# dilate the thresholded image to fill in holes, then find contours
# on thresholded image
thresh = cv2.dilate(thresh, None, iterations=2)
(cnts, _) = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# loop over the contours
for c in cnts:
# if the contour is too small, ignore it
if cv2.contourArea(c) < 20000:
continue
# compute the bounding box for the contour, draw it on the frame,
# and update the text
(x, y, w, h) = cv2.boundingRect(c)
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
text = "Found hand"
hand = True
cv2.putText(frame, "Room Status: {}".format(text), (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
cv2.imshow("Hand Search", frame)
key = cv2.waitKey(1) & 0xFF
if hand:
break
# if the `q` key is pressed, break from the lop
if key == ord("q"):
break
if hand:
learningFrames(vc)
handTracker = None
opts,args = cmd_parser()
inputMode,inTrainDirs = process_opts(opts)
vc = cv2.VideoCapture(0)
try:
#recognizer = Recognizer(vc=vc, opts=opts)
#score = recognizer.train_from_video()
#print "Training score = {0}".format(score)
#outTrainDir = get_new_directory(opts.num, opts.type)
# ret,frame1 = vc.read()
# ret,frame2 = vc.read()
# while True:
# cv2.imshow('img1', frame1)
# cv2.imshow('img2', frame2)
# if cv2.waitKey(1) & 0xFF == ord('y'): #save on pressing 'y'
# #cv2.imwrite('images/c1.png',frame)
# cv2.destroyAllWindows()
# break
r,f = vc.read()
handTracker = HandTracker(kernelSize=7, thresholdAngle=0.4, defectDistFromHull=30, parent=None, frame=f)
no_hand(vc)
vc.release()
cv2.destroyAllWindows()
except:
vc.release()
cv2.destroyAllWindows()
import traceback
traceback.print_exc(file=sys.stdout)
