class Responder:
def __init__(self):
self.classifiers = []
self.bound_methods = {}
print(
"Keys:\n"
" k - kill the program\n"
" r - reset the gesture classification\n"
" w - start/stop recording current pattern\n"
" s - save the patterns\n"
#" b - switch to/from backprojection view\n"
"To initialize tracking, drag across the object with the mouse\n")
def process_gesture(self, gest_name):
if gest_name in self.bound_methods:
self.bound_methods[gest_name]()
return
# adds a new classifier that will handle points
def add_classifier(self, classifier):
classifier.attach(self)
self.classifiers.append(classifier)
return
# starts the filtering loop
def start(self):
self.tracker = HandTracker(self.classifiers, debug=True)
self.tracker.start()
class Responder:
def __init__(self):
self.classifiers = []
self.bound_methods = {}
print( "Keys:\n"
" k - kill the program\n"
" r - reset the gesture classification\n"
" w - start/stop recording current pattern\n"
" s - save the patterns\n"
#" b - switch to/from backprojection view\n"
"To initialize tracking, drag across the object with the mouse\n" )
def process_gesture(self,gest_name):
if gest_name in self.bound_methods:
self.bound_methods[gest_name]()
return
# adds a new classifier that will handle points
def add_classifier(self,classifier):
classifier.attach(self)
self.classifiers.append(classifier)
return
# starts the filtering loop
def start(self):
self.tracker = HandTracker(self.classifiers,debug=True)
self.tracker.start()
def __init__(self):
threading.Thread.__init__(self)
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0,
box_enlarge=1.5)
def __init__(self, palm_recog_path, kp_recog_path, anchor_path):
self.detector = HandTracker(palm_recog_path,
kp_recog_path,
anchor_path,
box_shift=0.2,
box_enlarge=1.3)
self.finger_bend = [False] * 5
def __init__(self):
rospy.init_node('rgb_object_detection')
self.fps_reduce = 0
self.palm_model_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/models/palm_detection.tflite"
self.landmark_model_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/models/hand_landmark.tflite"
self.anchors_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/data/anchors.csv"
self.estimator_path = "/home/pavel/ros_workspace/cv_tutorial/nodes/py2-RPS/logregest_1200.pickle"
self.bridge = CvBridge()
self.estimator = self.get_estimator()
self.detector = HandTracker(self.palm_model_path,
self.landmark_model_path,
self.anchors_path,
box_shift=0.2,
box_enlarge=1.3)
self.start_f = False
self.under_line = False
self.end_flag = False
self.count_moves = 0
self.delay_counter = 0
self.end_frame = 0
self.img_sub = rospy.Subscriber("/camera/color/image_raw",
Image,
self.img_to_cv2,
queue_size=1)
self.res_pub = rospy.Publisher("/RPS_node/result",
String,
queue_size=1)
self.res_pub.publish("playing a b")
def __init__(self):
# Initializing the HandTracker #################################################
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.1,
box_enlarge=1.5)
self.kp_single_int = tuple([0, 0]) # initialized as the result
# Setting keypoints pairs ###############################
# self.pair_WRONG = [[1, 2], [2, 3], [3, 4],
# [5, 6], [6, 7], [7, 8],
# [9, 10], [10, 11], [11, 12],
# [13, 14], [14, 15], [15, 16],
# [17, 18], [18, 19],
# [19, 20]] # Adding a WRONG at the end of function name just means this is a
# # wrong way of calculation
self.pair = [[0, 1, 2], [1, 2, 3], [2, 3, 4], [0, 5, 6], [5, 6, 7],
[6, 7, 8], [0, 9, 10], [9, 10, 11], [10, 11, 12],
[0, 13, 14], [13, 14, 15], [14, 15, 16], [0, 17, 18],
[17, 18, 19], [18, 19, 20]]
self.stem_pair = [0, 9] # palm points and root middle-finger point
def __init__(self, numGestures, numFramesPerGesture,
minDescriptorsPerFrame, numWords, descType, kernel, numIter,
parent):
self.numGestures = numGestures
self.numFramesPerGesture = numFramesPerGesture
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.desList = []
self.voc = None
self.classifier = None
self.windowName = "Training preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.kernel = kernel
self.numIter = numIter
self.numDefects = None
self.firstFrameList = []
self.trainLabels = []
def __init__(self, numGestures, minDescriptorsPerFrame, numWords, descType,
numPredictions, parent):
self.numGestures = numGestures
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.classifier = None
self.windowName = "Testing preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.predictionList = [-1] * numPredictions
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.numSideFrames = 10
self.prevFrameList = np.zeros(
(self.numSideFrames, self.parent.imHeight / self.numSideFrames,
self.parent.imWidth / self.numSideFrames, 3), "uint8")
self.numPrevFrames = 0
self.predictionScoreThreshold = 0.2
self.learningRate = 0.01
self.numReinforce = 1
def __init__(self):
# Initializing the HandTracker #################################################
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
self.detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.1,
box_enlarge=1.5)
self.kp_single_int = tuple([0, 0]) # initialized as the result
# Setting keypoints pairs ###############################
self.pair = [[1, 2], [2, 3], [3, 4], [5, 6], [6, 7], [7, 8], [9, 10],
[10, 11], [11, 12], [13, 14], [14, 15], [15, 16],
[17, 18], [18, 19], [19, 20]]
self.stem_pair = [0, 9] # palm points and root middle-finger point
def __init__(self, numGestures, minDescriptorsPerFrame, numWords, descType, numPredictions, parent):
self.numGestures = numGestures
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.classifier = None
self.windowName = "Testing preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.predictionList = [-1]*numPredictions;
self.handTracker = HandTracker(kernelSize=7, thresholdAngle=0.4, defectDistFromHull=30, parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.numSideFrames = 10
self.prevFrameList = np.zeros((self.numSideFrames,self.parent.imHeight/self.numSideFrames,self.parent.imWidth/self.numSideFrames,3), "uint8")
self.numPrevFrames = 0
self.predictionScoreThreshold = 0.2
self.learningRate = 0.01
self.numReinforce = 1
def main(args):
det = HandTracker('models/palm_detection_without_custom_op.tflite',
'models/hand_landmark_3d.tflite',
'data/anchors.csv',
box_shift=-0.5,
box_enlarge=2.6)
video_name = args.video
out_dir = args.outdir
print('Processing {}'.format(video_name))
for frame_i, img in enumerate(read_video(video_name)):
img_rgb = img[:, :, ::-1]
kpts, boxs = det(img_rgb)
out_file = out_dir + '/' + str(frame_i) + '.jpg'
if boxs is None:
out_img = img
else:
out_img = draw_hands(img, kpts, boxs)
cv2.imwrite(out_file, out_img)
def start(self): self.tracker = HandTracker(self.classifiers,debug=True) self.tracker.start()
class Tester(object):
def __init__(self, numGestures, minDescriptorsPerFrame, numWords, descType, numPredictions, parent):
self.numGestures = numGestures
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.classifier = None
self.windowName = "Testing preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.predictionList = [-1]*numPredictions;
self.handTracker = HandTracker(kernelSize=7, thresholdAngle=0.4, defectDistFromHull=30, parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.numSideFrames = 10
self.prevFrameList = np.zeros((self.numSideFrames,self.parent.imHeight/self.numSideFrames,self.parent.imWidth/self.numSideFrames,3), "uint8")
self.numPrevFrames = 0
self.predictionScoreThreshold = 0.2
self.learningRate = 0.01
self.numReinforce = 1
def initialize(self, clf):
self.classifier = clf
self.numWords = self.classifier.voc.shape[0]
self.prevStates = np.zeros((self.numSideFrames, self.numWords), "float32")
self.prevLabels = [0]*self.numSideFrames
self.prevScores = [0]*self.numSideFrames
def test_on_video(self):
vc = self.parent.vc
while(vc.isOpened()):
ret,im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
self.handTracker.colorProfiler.draw_color_windows(im, imhsv)
cv2.imshow(self.windowName, im)
k = cv2.waitKey(1)
if k == 32: # space
break
elif k == 27:
sys.exit(0)
self.handTracker.colorProfiler.run()
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt,hull,centroid,defects = self.handTracker.initialize_contour(binaryIm)
cv2.namedWindow(self.binaryWindowName)
cv2.namedWindow(self.handWindowName)
cv2.namedWindow(self.windowName)
cv2.setMouseCallback(self.windowName, self.reinforce)
while(vc.isOpened()):
ret,im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt,hull,centroid,defects = self.handTracker.get_contour(binaryIm)
imCopy = 1*im
testData = None
prediction = -1
score = -1
update = False
if cnt is not None:
numDefects = defects.shape[0]
cropImage,cropPoints = self.handTracker.get_cropped_image_from_cnt(im, cnt, 0.05)
cropImageGray = self.handTracker.get_cropped_image_from_points(imgray, cropPoints)
#cv2.fillPoly(binaryIm, cnt, 255)
#cropImageBinary = self.handTracker.get_cropped_image_from_points(binaryIm, cropPoints)
#cropImageGray = self.apply_binary_mask(cropImageGray, cropImageBinary, 5)
#kp,des = self.featureExtractor.get_keypoints_and_descriptors(cropImageGray)
kp = self.featureExtractor.get_keypoints(cropImageGray)
cropCnt = self.handTracker.get_cropped_contour(cnt, cropPoints)
kp = self.featureExtractor.get_keypoints_in_contour(kp, cropCnt)
kp,des = self.featureExtractor.compute_descriptors(cropImageGray, kp)
if des is not None and des.shape[0] >= 0:
self.featureExtractor.draw_keypoints(cropImage, kp)
if des is not None and des.shape[0] >= self.minDescriptorsPerFrame and self.is_hand(defects):
words, distance = vq(des, self.classifier.voc)
testData = np.zeros(self.numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(self.numWords)
testData = np.divide(testData, normTestData)
prediction,score = self.predict(testData)
sortedScores = np.sort(score)
#if max(score) > self.predictionScoreThreshold:
if sortedScores[-1]-sortedScores[-2] >= self.predictionScoreThreshold:
self.handTracker.draw_on_image(imCopy, cnt=False, hullColor=(0,255,0))
else:
self.handTracker.draw_on_image(imCopy, cnt=False, hullColor=(255,0,0))
prediction = -1
update = True
else:
self.handTracker.draw_on_image(imCopy, cnt=False, hullColor=(0,0,255))
prediction = -1
cv2.imshow(self.handWindowName,cropImage)
else:
prediction = -1
#self.insert_to_prediction_list(prediction)
#prediction,predictionCount = self.most_common(self.predictionList)
#if prediction>=0:
writtenVal = '-'
if prediction > 0:
#if self.classifier.medianDefects is not None and numDefects>=self.classifier.medianDefects[prediction-1]-1 and numDefects<=self.classifier.medianDefects[prediction-1]+1:
# #print prediction
# writtenVal = str(prediction)
# update = True
#elif self.classifier.medianDefects is None:
#print prediction
writtenVal = str(prediction)
self.write_on_image(imCopy, writtenVal)
cv2.imshow(self.binaryWindowName, binaryIm)
imCopy = self.add_prev_frames_to_image(imCopy, testData, prediction, score, update)
cv2.imshow(self.windowName,imCopy)
k = cv2.waitKey(1)
if k == 27: # space
break
def test_on_descriptors(self, desList):
testLabels = []
for i,des in enumerate(desList):
if des is not None and des.shape[0] >= self.minDescriptorsPerFrame:
words, distance = vq(des, self.classifier.voc)
testData = np.zeros(self.numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(self.numWords)
testData = np.divide(testData, normTestData)
prediction,score = self.predict(testData)
sortedScores = np.sort(score)
#if max(score) > self.predictionScoreThreshold:
if sortedScores[-1]-sortedScores[-2] >= self.predictionScoreThreshold:
pass
else:
prediction = -1
else:
prediction = -1
testLabels.append(prediction)
return testLabels
def predict(self, testData):
prediction = self.classifier.predict(testData.reshape(1,-1))
score = self.classifier.decision_function(testData.reshape(1,-1))
return prediction[0], score[0]
def insert_to_prediction_list(self, prediction):
self.predictionList.append(prediction)
self.predictionList = self.predictionList[1:]
def most_common(self, lst):
for i in range(1,len(lst)-1):
if lst[i] != lst[i-1] and lst[i] != lst[i+1]:
lst[i] = -1
e = max(set(lst), key=lst.count)
return e,lst.count(e)
def is_hand(self, defects):
if defects.shape[0] > 5:
return False
else:
return True
def write_on_image(self, image, text):
cv2.putText(image, text, (self.parent.imWidth/20,self.parent.imHeight/4), cv2.FONT_HERSHEY_SIMPLEX, 5, (0,0,255), 5)
def get_prev_frames_image(self):
image = self.prevFrameList[0]
for i in range(1,len(self.prevFrameList)):
image = np.append(image, self.prevFrameList[i], axis=0)
return image
def apply_binary_mask(self, image, mask, kernelSize):
kernel = np.ones((kernelSize,kernelSize),np.uint8)
dilatedMask = cv2.dilate(mask,kernel,iterations=1)
maskedImage = cv2.bitwise_and(image, image, mask=dilatedMask)
return maskedImage
def add_prev_frames_to_image(self, image, testData, testLabel, testScore, update=False):
shrinkIm = cv2.resize(image, None, fx=float(1)/self.numSideFrames, fy=float(1)/self.numSideFrames)
prevFramesIm = self.get_prev_frames_image()
image = np.append(image, prevFramesIm, axis=1)
if update:
if self.numPrevFrames < self.numSideFrames:
self.prevFrameList[self.numPrevFrames] = shrinkIm
self.prevStates[self.numPrevFrames] = testData
self.prevLabels[self.numPrevFrames] = testLabel
self.prevScores[self.numPrevFrames] = testScore
self.numPrevFrames += 1
else:
self.prevFrameList = np.append(self.prevFrameList, np.array([shrinkIm]), axis=0)
self.prevFrameList = self.prevFrameList[1:]
self.prevStates = np.append(self.prevStates, np.array([testData]), axis=0)
self.prevStates = self.prevStates[1:]
self.prevLabels.append(testLabel)
self.prevLabels = self.prevLabels[1:]
self.prevScores.append(testScore)
self.prevScores = self.prevScores[1:]
return image
def reinforce(self, event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
if x > self.parent.imWidth:
prevFrameID = int(np.floor(y*self.numSideFrames/self.parent.imHeight))
self.prevFrameList[prevFrameID] = cv2.cvtColor(self.prevFrameList[prevFrameID], cv2.COLOR_BGR2HSV)
if isinstance(self.classifier, svm.LinearSVC):
self.perceptron_update(prevFrameID, False)
elif event == cv2.EVENT_RBUTTONDOWN:
if x > self.parent.imWidth:
prevFrameID = int(np.floor(y*self.numSideFrames/self.parent.imHeight))
self.prevFrameList[prevFrameID] = cv2.cvtColor(self.prevFrameList[prevFrameID], cv2.COLOR_BGR2YCR_CB)
if isinstance(self.classifier, svm.LinearSVC):
self.perceptron_update(prevFrameID, True)
def perceptron_update(self, prevFrameID, flag):
weights = self.classifier.coef_
if not flag:
wrongData = self.prevStates[prevFrameID]
#normData = np.linalg.norm(wrongData, ord=2) * np.ones(self.numWords)
#wrongData = np.divide(wrongData, normData)
wrongLabel = self.prevLabels[prevFrameID]
wrongScores = self.prevScores[prevFrameID]
wrongScore = max(wrongScores)
if wrongLabel > 0:
wrongWeights = weights[wrongLabel-1]
newWeights = np.subtract(wrongWeights, (self.learningRate/self.numReinforce)*wrongData)
weights[wrongLabel-1] = newWeights
else:
k = cv2.waitKey(-1)
rightLabel = k - 48
if rightLabel > 0 and rightLabel <= weights.shape[0]:
wrongWeights = weights[rightLabel-1]
newWeights = np.add(wrongWeights, (self.learningRate/self.numReinforce)*wrongData)
weights[rightLabel-1] = newWeights
else:
rightData = self.prevStates[prevFrameID]
#normData = np.linalg.norm(rightData, ord=2) * np.ones(self.numWords)
#rightData = np.divide(rightData, normData)
rightLabel = self.prevLabels[prevFrameID]
rightScores = self.prevScores[prevFrameID]
rightScore = max(rightScores)
if rightLabel > 0:
rightWeights = weights[rightLabel-1]
newWeights = np.add(rightWeights, (self.learningRate/self.numReinforce)*rightData)
weights[rightLabel-1] = newWeights
#self.numReinforce += 1
self.classifier.coef_ = weights
class Tester(object):
def __init__(self, numGestures, minDescriptorsPerFrame, numWords, descType,
numPredictions, parent):
self.numGestures = numGestures
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.classifier = None
self.windowName = "Testing preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.predictionList = [-1] * numPredictions
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.numSideFrames = 10
self.prevFrameList = np.zeros(
(self.numSideFrames, self.parent.imHeight / self.numSideFrames,
self.parent.imWidth / self.numSideFrames, 3), "uint8")
self.numPrevFrames = 0
self.predictionScoreThreshold = 0.2
self.learningRate = 0.01
self.numReinforce = 1
def initialize(self, clf):
self.classifier = clf
self.numWords = self.classifier.voc.shape[0]
self.prevStates = np.zeros((self.numSideFrames, self.numWords),
"float32")
self.prevLabels = [0] * self.numSideFrames
self.prevScores = [0] * self.numSideFrames
def test_on_video(self):
vc = self.parent.vc
while (vc.isOpened()):
ret, im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
self.handTracker.colorProfiler.draw_color_windows(im, imhsv)
cv2.imshow(self.windowName, im)
k = cv2.waitKey(1)
if k == 32: # space
break
elif k == 27:
sys.exit(0)
self.handTracker.colorProfiler.run()
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt, hull, centroid, defects = self.handTracker.initialize_contour(
binaryIm)
cv2.namedWindow(self.binaryWindowName)
cv2.namedWindow(self.handWindowName)
cv2.namedWindow(self.windowName)
cv2.setMouseCallback(self.windowName, self.reinforce)
while (vc.isOpened()):
ret, im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt, hull, centroid, defects = self.handTracker.get_contour(
binaryIm)
imCopy = 1 * im
testData = None
prediction = -1
score = -1
update = False
if cnt is not None:
numDefects = defects.shape[0]
cropImage, cropPoints = self.handTracker.get_cropped_image_from_cnt(
im, cnt, 0.05)
cropImageGray = self.handTracker.get_cropped_image_from_points(
imgray, cropPoints)
#cv2.fillPoly(binaryIm, cnt, 255)
#cropImageBinary = self.handTracker.get_cropped_image_from_points(binaryIm, cropPoints)
#cropImageGray = self.apply_binary_mask(cropImageGray, cropImageBinary, 5)
#kp,des = self.featureExtractor.get_keypoints_and_descriptors(cropImageGray)
kp = self.featureExtractor.get_keypoints(cropImageGray)
cropCnt = self.handTracker.get_cropped_contour(cnt, cropPoints)
kp = self.featureExtractor.get_keypoints_in_contour(
kp, cropCnt)
kp, des = self.featureExtractor.compute_descriptors(
cropImageGray, kp)
if des is not None and des.shape[0] >= 0:
self.featureExtractor.draw_keypoints(cropImage, kp)
if des is not None and des.shape[
0] >= self.minDescriptorsPerFrame and self.is_hand(
defects):
words, distance = vq(des, self.classifier.voc)
testData = np.zeros(self.numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(
self.numWords)
testData = np.divide(testData, normTestData)
prediction, score = self.predict(testData)
sortedScores = np.sort(score)
#if max(score) > self.predictionScoreThreshold:
if sortedScores[-1] - sortedScores[
-2] >= self.predictionScoreThreshold:
self.handTracker.draw_on_image(imCopy,
cnt=False,
hullColor=(0, 255, 0))
else:
self.handTracker.draw_on_image(imCopy,
cnt=False,
hullColor=(255, 0, 0))
prediction = -1
update = True
else:
self.handTracker.draw_on_image(imCopy,
cnt=False,
hullColor=(0, 0, 255))
prediction = -1
cv2.imshow(self.handWindowName, cropImage)
else:
prediction = -1
#self.insert_to_prediction_list(prediction)
#prediction,predictionCount = self.most_common(self.predictionList)
#if prediction>=0:
writtenVal = '-'
if prediction > 0:
#if self.classifier.medianDefects is not None and numDefects>=self.classifier.medianDefects[prediction-1]-1 and numDefects<=self.classifier.medianDefects[prediction-1]+1:
# #print prediction
# writtenVal = str(prediction)
# update = True
#elif self.classifier.medianDefects is None:
#print prediction
writtenVal = str(prediction)
self.write_on_image(imCopy, writtenVal)
cv2.imshow(self.binaryWindowName, binaryIm)
imCopy = self.add_prev_frames_to_image(imCopy, testData,
prediction, score, update)
cv2.imshow(self.windowName, imCopy)
k = cv2.waitKey(1)
if k == 27: # space
break
def test_on_descriptors(self, desList):
testLabels = []
for i, des in enumerate(desList):
if des is not None and des.shape[0] >= self.minDescriptorsPerFrame:
words, distance = vq(des, self.classifier.voc)
testData = np.zeros(self.numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(
self.numWords)
testData = np.divide(testData, normTestData)
prediction, score = self.predict(testData)
sortedScores = np.sort(score)
#if max(score) > self.predictionScoreThreshold:
if sortedScores[-1] - sortedScores[
-2] >= self.predictionScoreThreshold:
pass
else:
prediction = -1
else:
prediction = -1
testLabels.append(prediction)
return testLabels
def predict(self, testData):
prediction = self.classifier.predict(testData.reshape(1, -1))
score = self.classifier.decision_function(testData.reshape(1, -1))
return prediction[0], score[0]
def insert_to_prediction_list(self, prediction):
self.predictionList.append(prediction)
self.predictionList = self.predictionList[1:]
def most_common(self, lst):
for i in range(1, len(lst) - 1):
if lst[i] != lst[i - 1] and lst[i] != lst[i + 1]:
lst[i] = -1
e = max(set(lst), key=lst.count)
return e, lst.count(e)
def is_hand(self, defects):
if defects.shape[0] > 5:
return False
else:
return True
def write_on_image(self, image, text):
cv2.putText(image, text,
(self.parent.imWidth / 20, self.parent.imHeight / 4),
cv2.FONT_HERSHEY_SIMPLEX, 5, (0, 0, 255), 5)
def get_prev_frames_image(self):
image = self.prevFrameList[0]
for i in range(1, len(self.prevFrameList)):
image = np.append(image, self.prevFrameList[i], axis=0)
return image
def apply_binary_mask(self, image, mask, kernelSize):
kernel = np.ones((kernelSize, kernelSize), np.uint8)
dilatedMask = cv2.dilate(mask, kernel, iterations=1)
maskedImage = cv2.bitwise_and(image, image, mask=dilatedMask)
return maskedImage
def add_prev_frames_to_image(self,
image,
testData,
testLabel,
testScore,
update=False):
shrinkIm = cv2.resize(image,
None,
fx=float(1) / self.numSideFrames,
fy=float(1) / self.numSideFrames)
prevFramesIm = self.get_prev_frames_image()
image = np.append(image, prevFramesIm, axis=1)
if update:
if self.numPrevFrames < self.numSideFrames:
self.prevFrameList[self.numPrevFrames] = shrinkIm
self.prevStates[self.numPrevFrames] = testData
self.prevLabels[self.numPrevFrames] = testLabel
self.prevScores[self.numPrevFrames] = testScore
self.numPrevFrames += 1
else:
self.prevFrameList = np.append(self.prevFrameList,
np.array([shrinkIm]),
axis=0)
self.prevFrameList = self.prevFrameList[1:]
self.prevStates = np.append(self.prevStates,
np.array([testData]),
axis=0)
self.prevStates = self.prevStates[1:]
self.prevLabels.append(testLabel)
self.prevLabels = self.prevLabels[1:]
self.prevScores.append(testScore)
self.prevScores = self.prevScores[1:]
return image
def reinforce(self, event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONDOWN:
if x > self.parent.imWidth:
prevFrameID = int(
np.floor(y * self.numSideFrames / self.parent.imHeight))
self.prevFrameList[prevFrameID] = cv2.cvtColor(
self.prevFrameList[prevFrameID], cv2.COLOR_BGR2HSV)
if isinstance(self.classifier, svm.LinearSVC):
self.perceptron_update(prevFrameID, False)
elif event == cv2.EVENT_RBUTTONDOWN:
if x > self.parent.imWidth:
prevFrameID = int(
np.floor(y * self.numSideFrames / self.parent.imHeight))
self.prevFrameList[prevFrameID] = cv2.cvtColor(
self.prevFrameList[prevFrameID], cv2.COLOR_BGR2YCR_CB)
if isinstance(self.classifier, svm.LinearSVC):
self.perceptron_update(prevFrameID, True)
def perceptron_update(self, prevFrameID, flag):
weights = self.classifier.coef_
if not flag:
wrongData = self.prevStates[prevFrameID]
#normData = np.linalg.norm(wrongData, ord=2) * np.ones(self.numWords)
#wrongData = np.divide(wrongData, normData)
wrongLabel = self.prevLabels[prevFrameID]
wrongScores = self.prevScores[prevFrameID]
wrongScore = max(wrongScores)
if wrongLabel > 0:
wrongWeights = weights[wrongLabel - 1]
newWeights = np.subtract(
wrongWeights,
(self.learningRate / self.numReinforce) * wrongData)
weights[wrongLabel - 1] = newWeights
else:
k = cv2.waitKey(-1)
rightLabel = k - 48
if rightLabel > 0 and rightLabel <= weights.shape[0]:
wrongWeights = weights[rightLabel - 1]
newWeights = np.add(
wrongWeights,
(self.learningRate / self.numReinforce) * wrongData)
weights[rightLabel - 1] = newWeights
else:
rightData = self.prevStates[prevFrameID]
#normData = np.linalg.norm(rightData, ord=2) * np.ones(self.numWords)
#rightData = np.divide(rightData, normData)
rightLabel = self.prevLabels[prevFrameID]
rightScores = self.prevScores[prevFrameID]
rightScore = max(rightScores)
if rightLabel > 0:
rightWeights = weights[rightLabel - 1]
newWeights = np.add(rightWeights,
(self.learningRate / self.numReinforce) *
rightData)
weights[rightLabel - 1] = newWeights
#self.numReinforce += 1
self.classifier.coef_ = weights
# 4 | | | |
# | 6 10 14 18
# 3 | | | |
# | 5---9---13--17
# 2 \ /
# \ \ /
# 1 \ /
# \ \ /
# ------0-
connections = [(0, 1), (1, 2), (2, 3), (3, 4), (5, 6), (6, 7), (7, 8), (9, 10),
(10, 11), (11, 12), (13, 14), (14, 15), (15, 16), (17, 18),
(18, 19), (19, 20), (0, 5), (5, 9), (9, 13), (13, 17), (0, 17)]
detector = HandTracker(PALM_MODEL_PATH,
LANDMARK_MODEL_PATH,
ANCHORS_PATH,
box_shift=0.2,
box_enlarge=1.3)
def runModel(img, threshold):
image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (256, 256))
points = detector(image, threshold)
output_json = list()
for points_ in points:
if points_ is not None:
#save outputs into json
model_output_json = {}
import sys
import numpy as np
sys.path.append("/api")
from pose_util import hand_from_raw
sys.path.append("/hand_tracking")
from hand_tracker import HandTracker
palm_model_path = "/hand_tracking/models/palm_detection.tflite"
landmark_model_path = "/hand_tracking/models/hand_landmark_3d.tflite"
anchors_path = "/hand_tracking/data/anchors.csv"
detector = HandTracker(palm_model_path,
landmark_model_path,
anchors_path,
box_shift=0.2,
box_enlarge=1.3)
reader = imageio.get_reader('/video.mp4')
counter = 0
for i, img in enumerate(reader):
counter += 1
if img.shape[0] == 256 and img.shape[1] == 256:
print("Skipping hand detection")
hands = [{"bbox": np.array([(0, 256), (256, 256), (256, 0), (0, 0)])}]
hands = [detector.get_landmarks(img, h) for h in hands]
else:
hands = detector(img)
def start(self):
self.tracker = HandTracker(self.classifiers, debug=True)
self.tracker.start()
from matplotlib.patches import Polygon
from hand_tracker import HandTracker
from scipy.signal import savgol_filter
import cv2
import numpy as np
import pickle
import sys
import glob
import os
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
detector = HandTracker(palm_model_path, landmark_model_path, anchors_path)
def write_kps(infile, outfile):
cap = cv2.VideoCapture(infile)
data = []
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_counter = 0
while True:
res, frame = cap.read()
if not res:
if frame_counter == total_frames:
print('finished, writing output to', outfile)
pickle.dump(data, open(outfile, 'wb'))
else:
import random
import sys
import argparse
print("you must need webcam to run this code....")
option=input("y or n... ")
if option=="n" or option!="y":
sys.exit()
palm_model_path = "./models/palm_detection.tflite"
landmark_model_path = "./models/hand_landmark.tflite"
anchors_path = "./data/anchors.csv"
# box_shift determines
detector = HandTracker(palm_model_path, landmark_model_path, anchors_path,
box_shift=0.2, box_enlarge=1.3)
def finger_det():
print("place ur right hand with gesture shows in picture in front of web cam..... ")
print("image will click after countdown three....")
time.sleep(3)
i=3
while i>0:
print(i)
time.sleep(1)
i-=1
class CursorController:
def __init__(self, options):
self.options = options
self.hand_tracker = HandTracker(self.options)
self.filter = Filter2D()
self.hand_prev = None
self.idle_frames = 0
def update(self, frame):
screen = (win32api.GetSystemMetrics(1), win32api.GetSystemMetrics(0))
hand_rect = self.hand_tracker.get_hand_rect(frame)
if hand_rect is not None:
cv2.rectangle(frame, (int(hand_rect[0]), int(hand_rect[1])),
(int(hand_rect[2]), int(hand_rect[3])), (0, 255, 0))
hand = util.to_relative(hand_rect, frame.shape)
pos = util.rect_center(hand)
if self.hand_prev is not None:
pos_prev = util.rect_center(self.hand_prev)
delta = [pos[0] - pos_prev[0], pos[1] - pos_prev[1]]
delta = self.filter.update(delta)
v0 = util.vec_len(delta)
if v0 < self.options[consts.vmin]:
v = v0 * v0 / self.options[consts.vmin]
elif v0 > self.options[consts.vmax]:
v = v0 * v0 / self.options[consts.vmax]
else:
v = v0
if v < self.options[consts.vidle]:
self.idle_frames += 1
if self.idle_frames == self.options[consts.idle_frames]:
cursor = win32api.GetCursorPos()
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,
cursor[0], cursor[1], 0, 0)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,
cursor[0], cursor[1], 0, 0)
self.idle_frames = 0
else:
self.idle_frames = 0
if v0 > 0:
depth_k = self.options[consts.depth_ratio] / (hand[3] -
hand[1])
sensetivity_x = self.options[
consts.sensetivity_x] * depth_k
sensetivity_y = self.options[
consts.sensetivity_y] * depth_k
offset = (float(screen[1]) * delta[0] / v0 * v *
sensetivity_x, float(screen[0]) * delta[1] / v0 *
v * sensetivity_y)
cursor0 = win32api.GetCursorPos()
cursor = (
cursor0[0] + offset[0],
cursor0[1] + offset[1],
)
win32api.SetCursorPos(
(int(cursor[0] + 0.5), int(cursor[1] + 0.5)))
self.hand_prev = hand
else:
kx = 1.0
ky = 1.0
cursor_rel = [
0.5 + (pos[1] - 0.5) * ky, 0.5 + (pos[0] - 0.5) * kx
]
cursor = (int(float(screen[1]) * cursor_rel[0] + 0.5),
int(float(screen[0]) * cursor_rel[1] + 0.5))
win32api.SetCursorPos(cursor)
self.hand_prev = hand
self.filter.reset()
else:
self.hand_prev = None
def __init__(self, options):
self.options = options
self.hand_tracker = HandTracker(self.options)
self.filter = Filter2D()
self.hand_prev = None
self.idle_frames = 0
def learningFrames(vc):
t = 0
while(vc.isOpened()):
ret,im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
handTracker.colorProfiler.draw_color_windows(im, imhsv)
cv2.imshow("color", im)
k = cv2.waitKey(1)
if k == 32: # space
break
elif k == 27:
sys.exit(0)
handTracker.colorProfiler.run()
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
imhsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
binaryIM = handTracker.get_binary_image(imhsv)
cnt,hull,centroids, defects = handTracker.initialize_contour(binaryIM)
while True:
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
imhsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
imgray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
binaryIm = handTracker.get_binary_image(imhsv)
cnt,hull,centroids, defects = handTracker.get_contour(binaryIm)
frameCopy = 1*frame
testData = None
prediction = -1
score = -1
if cnt is not None:
numDefects = defects.shape[0]
cropImage,cropPoints = handTracker.get_cropped_image_from_cnt(frame, cnt, 0.05)
cropImageGray = handTracker.get_cropped_image_from_points(imgray, cropPoints)
kp = featureExtractor.get_keypoints(cropImageGray)
cropCnt = handTracker.get_cropped_contour(cnt, cropPoints)
kp = featureExtractor.get_keypoints_in_contour(kp, cropCnt)
kp, des = featureExtractor.compute_descriptors(cropImageGray, kp)
if des is not None and des.shape[0] >= 0:
featureExtractor.draw_keypoints(cropImage, kp)
des = np.float32(des)
if des is not None and des.shape[0] >= 10 and is_hand(defects):
voc, variance = kmeans(des, numWords, 30)
words, distance = vq(des, voc)
testData = np.zeros(numWords, "float32")
for w in words:
testData[w] += 1
normTestData = np.linalg.norm(testData, ord=2) * np.ones(numWords)
testData = np.divide(testData, normTestData)
states.append([testData, np.zeros(numWords), testData])
class_probabilities = clf.predict_proba(testData)
print t
print class_probabilities
if max(class_probabilities) > 0.3:
cv2.imshow("frames", cropImage)
t = t + 1
else:
handTracker.draw_on_image(frameCopy, cnt=False, hullColor=(0,0,255))
else:
prediction = -1
cv2.imshow('testing', frameCopy)
def no_hand(vc):
firstFrame = None
hand = False
while True:
ret, frame = vc.read()
frame = cv2.flip(frame, 1)
text = "Unoccupied"
hand = False
frame = imutils.resize(frame, width=500)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (21, 21), 0)
if firstFrame is None:
firstFrame = gray
continue
frameDelta = cv2.absdiff(firstFrame, gray)
thresh = cv2.threshold(frameDelta, 25, 255, cv2.THRESH_BINARY)[1]
# dilate the thresholded image to fill in holes, then find contours
# on thresholded image
thresh = cv2.dilate(thresh, None, iterations=2)
(cnts, _) = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# loop over the contours
for c in cnts:
# if the contour is too small, ignore it
if cv2.contourArea(c) < 20000:
continue
# compute the bounding box for the contour, draw it on the frame,
# and update the text
(x, y, w, h) = cv2.boundingRect(c)
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
text = "Found hand"
hand = True
cv2.putText(frame, "Room Status: {}".format(text), (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
cv2.imshow("Hand Search", frame)
key = cv2.waitKey(1) & 0xFF
if hand:
break
# if the `q` key is pressed, break from the lop
if key == ord("q"):
break
if hand:
learningFrames(vc)
handTracker = None
opts,args = cmd_parser()
inputMode,inTrainDirs = process_opts(opts)
vc = cv2.VideoCapture(0)
try:
#recognizer = Recognizer(vc=vc, opts=opts)
#score = recognizer.train_from_video()
#print "Training score = {0}".format(score)
#outTrainDir = get_new_directory(opts.num, opts.type)
# ret,frame1 = vc.read()
# ret,frame2 = vc.read()
# while True:
# cv2.imshow('img1', frame1)
# cv2.imshow('img2', frame2)
# if cv2.waitKey(1) & 0xFF == ord('y'): #save on pressing 'y'
# #cv2.imwrite('images/c1.png',frame)
# cv2.destroyAllWindows()
# break
r,f = vc.read()
handTracker = HandTracker(kernelSize=7, thresholdAngle=0.4, defectDistFromHull=30, parent=None, frame=f)
no_hand(vc)
vc.release()
cv2.destroyAllWindows()
except:
vc.release()
cv2.destroyAllWindows()
import traceback
traceback.print_exc(file=sys.stdout)
import numpy as np
import cv2
from hand_tracker import HandTracker
det = HandTracker('models/palm_detection_without_custom_op.tflite',
'models/hand_landmark_3d.tflite',
'data/anchors.csv',
box_shift=-0.5,
box_enlarge=2.6)
in_bgr = cv2.imread('data/test_img1.jpg')
in_rgb = in_bgr[:, :, ::-1]
list_keypoints, list_bbox = det(in_rgb)
out_img = np.copy(in_bgr)
# point size
ps = int(np.ceil(min(out_img.shape[0], out_img.shape[1]) / 256))
if list_keypoints is not None:
for idx in range(len(list_keypoints)):
keypoints = list_keypoints[idx]
bbox = list_bbox[idx]
for i in range(4):
j = (i + 1) % 4
p0 = (int(bbox[i, 0] + 0.5), int(bbox[i, 1] + 0.5))
p1 = (int(bbox[j, 0] + 0.5), int(bbox[j, 1] + 0.5))
cv2.line(out_img, p0, p1, (0, 0, 255), ps)
for i in range(keypoints.shape[0]):
p = (int(keypoints[i, 0] + 0.5), int(keypoints[i, 1] + 0.5))
cv2.circle(out_img, p, ps, (255, 0, 0), ps)
cv2.imwrite('out.jpg', out_img)
class Trainer(object):
def __init__(self, numGestures, numFramesPerGesture,
minDescriptorsPerFrame, numWords, descType, kernel, numIter,
parent):
self.numGestures = numGestures
self.numFramesPerGesture = numFramesPerGesture
self.numWords = numWords
self.minDescriptorsPerFrame = minDescriptorsPerFrame
self.parent = parent
self.desList = []
self.voc = None
self.classifier = None
self.windowName = "Training preview"
self.handWindowName = "Cropped hand"
self.binaryWindowName = "Binary frames"
self.handTracker = HandTracker(kernelSize=7,
thresholdAngle=0.4,
defectDistFromHull=30,
parent=self)
self.featureExtractor = FeatureExtractor(type=descType, parent=self)
self.kernel = kernel
self.numIter = numIter
self.numDefects = None
self.firstFrameList = []
self.trainLabels = []
def extract_descriptors_from_images(self, gestureDirList, parentDirPath,
trainMask, maskParentDirPath):
#self.numFramesPerGestureList = []
for i, gestureDir in enumerate(gestureDirList):
gestureDirPath = os.path.join(parentDirPath, gestureDir)
imgList = []
for dirpath, dirnames, filenames in os.walk("%s" %
(gestureDirPath),
topdown=True,
followlinks=True):
for f in filenames:
if f.endswith(".jpg"):
imgList.append(os.path.join(dirpath, f))
if trainMask != 0:
maskList = []
maskDirPath = os.path.join(maskParentDirPath, gestureDir)
for dirpath, dirnames, filenames in os.walk("%s" %
(maskDirPath),
topdown=True,
followlinks=True):
for f in filenames:
if f.endswith(".bmp"):
maskList.append(os.path.join(dirpath, f))
#self.numFramesPerGestureList.append(len(imgList))
gestureID = i + 1
for j, f in enumerate(imgList):
cropImage = cv2.imread(f)
cropImage = cv2.flip(cropImage, 1)
cropImageGray = cv2.cvtColor(cropImage, cv2.COLOR_BGR2GRAY)
kp = self.featureExtractor.get_keypoints(cropImageGray)
if trainMask != 0:
mask = cv2.imread(maskList[j])
mask = cv2.flip(mask, 1)
mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
if trainMask > 0:
ret, binaryIm = cv2.threshold(mask, 127, 255,
cv2.THRESH_BINARY)
else:
ret, binaryIm = cv2.threshold(mask, 127, 255,
cv2.THRESH_BINARY_INV)
binaryIm = cv2.dilate(binaryIm,
self.handTracker.kernel,
iterations=1)
cnt, hull, centroid, defects = self.handTracker.get_contour(
binaryIm, False)
kp = self.featureExtractor.get_keypoints_in_contour(
kp, cnt)
else:
kp = self.featureExtractor.get_keypoints(cropImageGray)
kp, des = self.featureExtractor.compute_descriptors(
cropImageGray, kp)
#kp,des = self.featureExtractor.get_keypoints_and_descriptors(cropImageGray)
if des is not None and des.shape[0] >= 0:
self.featureExtractor.draw_keypoints(cropImage, kp)
self.desList.append(des)
self.trainLabels.append(gestureID)
if j == 0:
self.firstFrameList.append(cropImage)
cv2.imshow(self.handWindowName, cropImage)
k = cv2.waitKey(1)
if k == 27:
sys.exit(0)
cv2.destroyAllWindows()
def extract_descriptors_from_video(self):
vc = self.parent.vc
while (vc.isOpened()):
ret, im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
self.handTracker.colorProfiler.draw_color_windows(im, imhsv)
cv2.imshow(self.windowName, im)
k = cv2.waitKey(1)
if k == 32: # space
break
elif k == 27:
sys.exit(0)
self.handTracker.colorProfiler.run()
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt, hull, centroid, defects = self.handTracker.initialize_contour(
binaryIm)
self.numDefects = np.zeros(
(self.numGestures, self.numFramesPerGesture), "uint8")
cv2.namedWindow(self.binaryWindowName)
cv2.namedWindow(self.handWindowName)
cv2.namedWindow(self.windowName)
#self.numFramesPerGestureList = [self.numFramesPerGesture] * self.numGestures
gestureID = 1
frameNum = 0
captureFlag = False
while (vc.isOpened()):
ret, im = vc.read()
im = cv2.flip(im, 1)
imhsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
binaryIm = self.handTracker.get_binary_image(imhsv)
cnt, hull, centroid, defects = self.handTracker.get_contour(
binaryIm)
imCopy = 1 * im
if cnt is not None:
cropImage, cropPoints = self.handTracker.get_cropped_image_from_cnt(
im, cnt, 0.05)
cropImageGray = self.handTracker.get_cropped_image_from_points(
imgray, cropPoints)
#cv2.fillPoly(binaryIm, cnt, 255)
#cropImageBinary = self.handTracker.get_cropped_image_from_points(binaryIm, cropPoints)
#cropImageGray = self.apply_binary_mask(cropImageGray, cropImageBinary, 5)
#kp,des = self.featureExtractor.get_keypoints_and_descriptors(cropImageGray)
kp = self.featureExtractor.get_keypoints(cropImageGray)
cropCnt = self.handTracker.get_cropped_contour(cnt, cropPoints)
kp = self.featureExtractor.get_keypoints_in_contour(
kp, cropCnt)
kp, des = self.featureExtractor.compute_descriptors(
cropImageGray, kp)
if des is not None and des.shape[0] >= 0:
self.featureExtractor.draw_keypoints(cropImage, kp)
#x = int(cropPoints[0])
#y = int(cropPoints[1])
#w = int(cropPoints[2])
#h = int(cropPoints[3])
#cv2.rectangle(imCopy,(x,y),(x+w,y+h),(0,255,0),2)
if captureFlag:
if frameNum == 0:
self.firstFrameList.append(im)
if des is not None and des.shape[
0] >= self.minDescriptorsPerFrame and self.is_hand(
defects):
self.desList.append(des)
self.trainLabels.append(gestureID)
self.handTracker.draw_on_image(imCopy,
cnt=False,
hullColor=(0, 255, 0))
self.numDefects[gestureID -
1][frameNum] = defects.shape[0]
frameNum += 1
else:
self.handTracker.draw_on_image(imCopy,
cnt=False,
hullColor=(0, 0, 255))
if frameNum >= self.numFramesPerGesture:
if gestureID >= self.numGestures:
break
else:
captureFlag = False
gestureID += 1
frameNum = 0
else:
self.handTracker.draw_on_image(imCopy, cnt=False)
cv2.imshow(self.handWindowName, cropImage)
if not captureFlag:
text = "Press <space> for new gesture {0}".format(gestureID)
else:
text = "Getting gesture {0}".format(gestureID)
self.write_on_image(imCopy, text)
cv2.imshow(self.binaryWindowName, binaryIm)
cv2.imshow(self.windowName, imCopy)
k = cv2.waitKey(1)
if not captureFlag:
#print "Press <space> for new gesture <{0}>!".format(gestureID)
if k == 32:
captureFlag = True
continue
if k == 27:
sys.exit(0)
elif k == 99:
cv2.imwrite("TrainingImage.jpg", imCopy)
cv2.imwrite("BinaryImage.jpg", binaryIm)
cv2.imwrite("CroppedImage.jpg", cropImage)
cv2.destroyAllWindows()
def apply_binary_mask(self, image, mask, kernelSize):
kernel = np.ones((kernelSize, kernelSize), np.uint8)
dilatedMask = cv2.dilate(mask, kernel, iterations=1)
maskedImage = cv2.bitwise_and(image, image, mask=dilatedMask)
return maskedImage
def kmeans(self):
print "Running k-means clustering with {0} iterations...".format(
self.numIter)
descriptors = self.desList[0]
for des in self.desList:
descriptors = np.vstack((descriptors, des))
if descriptors.dtype != "float32":
descriptors = np.float32(descriptors)
self.voc, variance = kmeans(descriptors, self.numWords, self.numIter)
return variance
def bow(self):
print "Extracting bag-of-words features for {0} visual words...".format(
self.numWords)
self.trainData = np.zeros((len(self.trainLabels), self.numWords),
"float32")
for i in range(len(self.trainLabels)):
words, distance = vq(self.desList[i], self.voc)
for w in words:
self.trainData[i][w] += 1
normTrainData = np.linalg.norm(self.trainData, ord=2,
axis=1) * np.ones((self.numWords, 1))
self.trainData = np.divide(self.trainData, normTrainData.T)
def svm(self):
print "Training SVM classifier with {0} kernel...".format(self.kernel)
if self.kernel == "linear":
clf = svm.LinearSVC()
elif self.kernel == "hist":
from sklearn.metrics.pairwise import additive_chi2_kernel
clf = svm.SVC(kernel=additive_chi2_kernel,
decision_function_shape='ovr')
else:
clf = svm.SVC(kernel=self.kernel,
decision_function_shape='ovr',
degree=2,
gamma=2)
valScore = self.leave_one_out_validate(clf)
clf.fit(self.trainData, self.trainLabels)
self.classifier = clf
self.classifier.voc = self.voc
# if self.numDefects is not None:
# self.classifier.medianDefects = np.median(self.numDefects, axis=1)
# else:
# self.classifier.medianDefects = None
return valScore
def leave_one_out_validate(self, clf):
fullTrainData = self.trainData
fullTrainLabels = self.trainLabels
accuracy = np.zeros(len(fullTrainLabels))
for i in range(len(fullTrainLabels)):
testData = fullTrainData[i]
testLabels = fullTrainLabels[i]
trainData = np.append(fullTrainData[:i],
fullTrainData[i + 1:],
axis=0)
trainLabels = np.append(fullTrainLabels[:i],
fullTrainLabels[i + 1:])
#clf = svm.LinearSVC()
clf.fit(trainData, trainLabels)
prediction = clf.predict(testData.reshape(1, -1))
#score = clf.decision_function(testData.reshape(1,-1))
if prediction != testLabels:
accuracy[i] = 0
else:
accuracy[i] = 1
return np.mean(accuracy)
def predict(self, testData):
prediction = self.classifier.predict(testData.reshape(1, -1))
score = self.classifier.decision_function(testData.reshape(1, -1))
return prediction[0], score[0]
def is_hand(self, defects):
if defects.shape[0] > 4:
return False
else:
return True
def write_on_image(self, image, text):
cv2.putText(image, text,
(self.parent.imWidth / 20, self.parent.imHeight / 10),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
