def move(self, prev, current, duration=0, block=False):
dx, dy = self.__dx__(prev, current), self.__dy__(prev, current)
GLog.d("dx= {}, dy= {}".format(dx, dy), tag=self.TAG)
pya.moveRel(xOffset=dx, yOffset=dy,
duration=duration) if block else pya.moveRel(xOffset=dx *
duration * 50,
yOffset=dy *
duration * 50,
duration=0.05)
def control(self, gesture, center=(0, 0), dt=0.2):
if (not self.__shouldConsidered__(gesture)):
GLog.d(
"gesture '{}' found but didn't considered due to sensitivity option"
.format(gesture),
tag=self.TAG)
return
self.__track__(gesture, center)
g_config = self.config[gesture] # config of this gesture
e1 = getTickCount()
# to detect:
# 'key down': config[gesture] == action and lastGesture != gesture
# 'key up': config[lastGesture] == action and lastGesture != gesture
# 'hold': config[gesture] == action
if (g_config["control"] == MOVE):
self.inputFeeder.move(g_config["prev"],
g_config["curr"],
duration=dt,
block=not self.feeding)
GLog.o(timeMessage("controlling", e1), tag="TIME")
elif (g_config["control"] == RIGHT_CLICK and self.lastGesture !=
gesture): # for first time only [press when 'key down']
GLog.o("right mouse click", tag=self.TAG)
self.inputFeeder.rightClick()
GLog.o(timeMessage("controlling", e1), tag="TIME")
elif (g_config["control"] == LEFT_CLICK
and self.lastGesture != gesture):
GLog.o("left mouse click", tag=self.TAG)
self.inputFeeder.leftClick()
GLog.o(timeMessage("controlling", e1), tag="TIME")
elif (g_config["control"] == DRAG_RIGHT_CLICK):
GLog.o("dragging with right click", tag=self.TAG)
self.inputFeeder.dragRightClick(g_config["prev"], g_config["curr"])
GLog.o(timeMessage("controlling", e1), tag="TIME")
elif (g_config["control"] == DRAG_LEFT_CLICK):
GLog.o("dragging with left click", tag=self.TAG)
self.inputFeeder.dragLeftClick(g_config["prev"], g_config["curr"])
GLog.o(timeMessage("controlling", e1), tag="TIME")
elif (g_config["control"] == ESCAPE and self.lastGesture != gesture):
GLog.o("escape press", tag=self.TAG)
self.inputFeeder.pressChars(['esc'])
GLog.o(timeMessage("controlling", e1), tag="TIME")
else:
GLog.d(
"gesture '{}' is not found in config file or already pressed".
format(gesture),
tag=self.TAG)
self.lastGesture = gesture
def control(self):
q = Queue()
q.get(block=True, timeout=0.01)
GLog.d("control started", tag=self.TAG)
while not self.stopped:
if (False and len(self.gestures) == 1):
gesture1, center1 = self.last
gesture2, center2 = self.gestures[-1]
center = []
if (gesture1 == gesture2):
#TODO c1 is always last center sent
#TODO c2 is constant, it is last value in the queue -> we here want to send the value that always evaluate to be
#TODO dx at the end same as in first time c2 - c1
#TODO To achieve it -> always set last as prev value sent to gc
#TODO so what we have now is c2 and lastVale (c1 in scd time)
#TODO we need to get what we will send to achieve this equation '? - lastValue = c2 - c1'
# for now only know the dir and send a small shift
smallShiftingX = 1 if center2[0] - center1[0] > 0 else -1
smallShiftingY = 1 if center2[1] - center1[1] > 0 else -1
center = center1[0] + smallShiftingX, center1[
1] + smallShiftingY #\? - c1
GLog.d("len is 1", tag=self.TAG)
self.gc.control(gesture1, center)
self.last = gesture1, center
if (len(self.gestures) == 0):
#todo: make it blocks instead of busy waiting --> Thread.join()
continue
else:
current = self.gestures.pop()
self.last = current
GLog.d("handling", tag=self.TAG)
self.gc.control(current['GST'], current['POS'],
current['duration'] / 2)
GLog.d("control end", tag=self.TAG)
def addGstRecognition(self, gst, center):
'''
Add new gesture recognition to the buffer of game controller
newGstRecognition: is a tuple of gst and center
'''
currTick = getTickCount()
timeElapsed = (currTick - self.prevTick) / getTickFrequency()
GLog.d(timeElapsed, tag='addGstRecognition: time elapsed')
self.prevTick = currTick
if (len(self.gestures) >= self.MAX_BUFFER_SIZE):
return False
inserted = {'GST': gst, 'POS': center, 'duration': timeElapsed}
self.gestures.insert(0, inserted)
return True
def fromFeatures(self, numDefects, lengthRatio, hullCntRatio, maxTwoCntRatio):
GLog.d("defects={}".format(numDefects), tag=self.TAG)
GLog.d("lengthRatio={}".format(lengthRatio), tag=self.TAG)
GLog.d("hullCntRatio={}".format(hullCntRatio), tag=self.TAG)
GLog.d("maxTwoCntRatio={}".format(maxTwoCntRatio), tag=self.TAG)
if numDefects >= 2:
return PALM
elif lengthRatio > 2:
return KNIFE
elif lengthRatio < 1.45 and lengthRatio > 0.8 and hullCntRatio > 0.8 and hullCntRatio < 1.2: # square like
return FIST
elif maxTwoCntRatio < 250 :
return ZERO
else:
return NO_GST
def recognize(self, roi, handMask):
try:
handMask = cv2.erode(handMask,np.ones((3,3)),iterations=3)
handMask = cv2.dilate(handMask,np.ones((3,3)),iterations=3)
handMask = roi * handMask
hand = handMask
hand = self.__preProcessing__(hand)
contours = self.__findContoursSorted__(hand, roi)
maxCnt = contours[0]
x,y,w,h = cv2.boundingRect(maxCnt)
croppedHand = hand[y:y+h, x:x+w]
contours = self.__findContoursSorted__(croppedHand, roi)
#maxCnt = contours[0]
maxCntArea = cv2.contourArea(maxCnt)
maxCnt2Area = cv2.contourArea(contours[1]) if len(contours) >=2 else 0.0001
f_maxTwoCntRatio = (maxCntArea - maxCnt2Area) / maxCnt2Area
hull = cv2.convexHull(maxCnt)
areahull = cv2.contourArea(hull)
f_hullCntRatio = areahull / maxCntArea
x,y,w,h = cv2.boundingRect(maxCnt)
cv2.rectangle(hand,(x,y),(x+w,y+h),(100),5) # * [debug]
f_lengthRatio = h / float(w)
f_defects = self.__findDefects__(roi, maxCnt)
ILog.o(croppedHand,'cropped hand')
ILog.d(hand,'hand in gesture recognizer')
return self.fromFeatures(f_defects, f_lengthRatio, f_hullCntRatio, f_maxTwoCntRatio), (x + int(w / 2), y + int(h / 2))#self.__contourCenter__(maxCnt)
except Exception as e:
GLog.d(e, tag=self.TAG)
return NO_GST, (0,0)
def scrollV(self, prev, current):
dy = self.__dy__(prev, current)
GLog.d("dy= {}".format(dy), tag=self.TAG)
pya.scroll(clicks=dy)
def scrollH(self, prev, current):
dx = self.__dx__(prev, current)
GLog.d("dx= {}".format(dx), tag=self.TAG)
pya.scroll(clicks=dx)
def dragRightClick(self, prev, current):
dx, dy = self.__dx__(prev, current), self.__dy__(prev, current)
GLog.d("dx= {}, dy= {}".format(dx, dy), tag=self.TAG)
pya.dragRel(xOffset=dx, yOffset=dy, duration=0.001, button='right')
def dragLeftClick(self, prev, current):
dx, dy = self.__dx__(prev, current), self.__dy__(prev, current)
GLog.d("dx= {}, dy= {}".format(dx, dy), tag=self.TAG)
pya.dragRel(xOffset=dx * 3, yOffset=dy * 3, duration=0.001)
def move(self, prev, current, duration = 0):
print('move')
dx, dy = self.__dx__(prev, current), self.__dy__(prev, current)
GLog.d("dx= {}, dy= {}".format(dx, dy), tag=self.TAG)
pya.moveRel(xOffset=dx*3, yOffset=dy*3, duration=duration)
def detect(self, frames_gray, currFrame_rgb):
e1 = cv2.getTickCount()
self.updateSkinBackgroundModel() # optimize
GLog.d(timeMessage('updateSkinBackgroundModel', e1), tag=self.TAG)
e1 = cv2.getTickCount()
roi, backgroundSubtraction = self.__motionDetection__.detect(
frames_gray, currFrame_rgb, self.getSkinBackgroundModel())
GLog.d(timeMessage('motionDetection', e1), tag=self.TAG)
#multiply by background or not (important)
e1 = cv2.getTickCount()
skinColorDetection = self.__skinModel__.detect(
currFrame_rgb) #* backgroundSubtraction
GLog.d(timeMessage('skinModel', e1), tag=self.TAG)
#Omar Trial
skinBackgroundModel = self.getSkinBackgroundModel()
#backgroundSubtraction += roi * skinBackgroundModel
#// morphologyWeight = calcMorphology(backgroundSubtraction) #* 0.1
e1 = cv2.getTickCount()
handOnly = self.handWithoutFace(currFrame_rgb, roi)
GLog.d(timeMessage('handWithoutFace', e1), tag=self.TAG)
e1 = cv2.getTickCount()
finalOut = self.__combine__(backgroundSubtraction, skinColorDetection,
None, skinBackgroundModel, handOnly)
GLog.d(timeMessage('combine', e1), tag=self.TAG)
e1 = cv2.getTickCount()
self.roi = roi
self.finalOut = finalOut
self.backgroundSubtraction = backgroundSubtraction
#// self.morphologyWeight = morphologyWeight
self.skinColorDetection = skinColorDetection
images, titles = self.getState()
ILog.d(images, titles)
GLog.d(timeMessage('detecthanddepyector', e1), tag=self.TAG)
return images, titles
