def getMetric(target, output, metric="CCC"):
if metric == "CCC":
result = CCC(target, output)
if metric == "MSE":
result = MSE(target, output)
if metric == "RMSE":
result = RMSE(target, output)
if metric == "Accuracy":
result = Accuracy(target, output)
if metric == "UAR":
from sklearn.metrics import recall_score
result = recall_score(target, output, average='macro')
if metric == "AUC":
from sklearn.metrics import roc_auc_score
result = roc_auc_score(target, output)
if "AUC-" in metric:
from sklearn.metrics import roc_auc_score
result = roc_auc_score(target, smooth(output, win=int(metric.split('-')[1])))
if "SmoothAcc-" in metric:
result = Accuracy(target, smooth(output, win=int(metric.split('-')[1])))
return result
if mouse_flg==1:
X=rpt[tipIndex][0]
mouseX=(X-600)*width/400
Y=rpt[tipIndex][1]
mouseY=(Y-150)*height/290
"""Currently we have the setting such that if there is a single LED that is out of range then
the mouse wont move. The problem with this is that the range of the mouse gets limited, and
some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice
versa. So this is a give or take until we have a better filtering method."""
if inrange:
buff[0].put(mouseX)
buff[1].put(mouseY)
smoothX=np.mean(fun.smooth(buff[0].data, window_len=len(buff[0].data)))
smoothY=np.mean(fun.smooth(buff[1].data, window_len=len(buff[1].data)))
m.move(buff[0].data[-1],buff[1].data[-1])
# m.move(smoothX,smoothY)
################################################################################################################
for event in pygame.event.get():
if event.type==KEYDOWN:
if event.key==pygame.K_r: #start recording
rec_flg=1
calibration=False
elif event.key==pygame.K_c: #start calibration
calibration=1
elif event.key==pygame.K_s: #pauses the recording
def run(self):
global FLG, coords
FLG = 1
m = PyMouse()
k = PyKeyboard()
#Intialization of Pygame
pygameRate=100
os.environ['SDL_VIDEO_iWINDOW_POS'] = "%d,%d" % (0,0)
pygame.mixer.pre_init(44100, -16, 2, 2048) # setup mixer to avoid sound lag
pygame.init()
vals.switchSound = pygame.mixer.Sound('switch.wav')
vals.clickSound = pygame.mixer.Sound('click.wav')
clock=pygame.time.Clock()
myfont=pygame.font.SysFont("monospace",15)
calibFont=pygame.font.SysFont("monospace",20)
depthFont=pygame.font.SysFont("monospace",10)
defaultFont=pygame.font.SysFont("monospace",15)
pygame.mixer.init()
infoObject = pygame.display.Info()
vals.width = infoObject.current_w
vals.height = infoObject.current_h
# print width, height
screen=pygame.display.set_mode((int(vals.width*0.5), vals.height))
global kill
#Main loop
while kill==False:
screen.fill((0,0,0))
rpt=[[int(i2) for i2 in i]for i in coords[0]]
rpt2=[[int(i2) for i2 in i]for i in coords[1]]
if vals.switchRPT:
tmp = rpt
rpt = rpt2
rpt2 = tmp
if not vals.rec_flg and (vals.calibration or vals.calibLoadFlag): #do calibration or load from file
#Receiving data from the threads
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
#GUI section
#doDraw.drawAllMiniCalibration(miniScreen, rpt, tipIndex, tipThumb,kThumb,kIndex,averageX,averageY,myfont,calibFont,depthFont)
doDraw.drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)
if vals.inputCalibration:
#Receiving data from the threads
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
#GUI section
doDraw.drawInputCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)
#first "click" get where the rpt index x,y, store it. Second click completes the line, turn into square
distClick=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
#first click
if (distClick[0]<vals.clickValue and vals.inputCounter==0):
vals.inputX1=rpt[tipIndex][0]
vals.inputY1=rpt[tipIndex][1]
vals.inputCounter+=1
vals.inputClickStopper=1
print "awef"
print vals.inputX1,vals.inputY1
if (distClick[0]>=vals.clickValue and vals.inputClickStopper):
vals.inputClickStopper=0
#second click
if (distClick[0]<vals.clickValue and vals.inputCounter==1 and not vals.inputClickStopper):
vals.inputX2=rpt[tipIndex][0]
vals.inputY2=rpt[tipIndex][1]
print "hey"
print vals.inputX2,vals.inputY2
vals.inputCounter+=1
vals.inputSet=1
if vals.inputSet:
#sets the inrange constants
if vals.inputX1<=vals.inputX2:
vals.leftBound=vals.inputX1
vals.rightBound=vals.inputX2
else:
vals.leftBound=vals.inputX2
vals.rightBound=vals.inputX1
if vals.inputY1<=vals.inputY2:
vals.upperBound=vals.inputY1
vals.lowerBound=vals.inputY2
else:
vals.upperBound=vals.inputY2
vals.lowerBound=vals.inputY1
vals.windowX = vals.rightBound - vals.leftBound
vals.windowY = vals.lowerBound - vals.upperBound
if vals.rec_flg==1: #Recording
#Finding out the location of the LEDs, tipThumb, kThumb....
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
averageX2,averageY2=findingPoints.centerFind(rpt2) #the center point
#Check whether LED is in range
newRpt=copy.deepcopy(rpt)
vals.rptList.append(newRpt)
vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
if (vals.inputX1<=rpt[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt[tipIndex][1]<=vals.inputY2) or \
(vals.inputX1<=rpt2[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt2[tipIndex][1]<=vals.inputY2):
vals.inrange=1
#Depth
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
#GUI
doDraw.drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,averageX2,averageY2,myfont,calibFont,depthFont)
#doDraw.drawAllMiniRecording(miniScreen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,myfont,calibFont,depthFont)
# Only log the depth data in record mode when testing or debugging
# if vals.relativeFlag:
# if vals.depthBuff[2].size() > 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# # log the depth and index tip raw coordinate
# vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
# smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
#Mouse Events
#doMouse.mouseActivities(pygame,rpt, tipIndex,tipThumb,kIndex,kThumb,m,k)
doMouse.clientMouseActivities(pygame,rpt,m,k)
#Gestures
# print doDepth.checkAllAboveBox()
# if doDepth.checkAllAboveBox():
# doGestures.gestures(averageX,averageY,k,m)
if vals.mouse_flg==1:
tipParam = 7
knuParam = 3
if vals.knuckleFlag:
fingerX = (rpt[tipIndex][0] * tipParam + rpt[kIndex][0] * knuParam) / (tipParam + knuParam)
fingerY = (rpt[tipIndex][1] * tipParam + rpt[kIndex][1] * knuParam) / (tipParam + knuParam)
else:
fingerX, fingerY = rpt[tipIndex][0], rpt[tipIndex][1]
# Relative:
if vals.relativeFlag:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
keyboardTest = myIsOnKeyboard(fingerX, fingerY, smoothTipIndex)
keyboardTest = isOnKeyboard(fingerX, fingerY, smoothTipIndex)
if vals.onKeyboardFlag and not keyboardTest:
# Hands up
vals.fingerBuff[0].erase()
vals.fingerBuff[1].erase()
# else:
# # Hands down
# pass
vals.onKeyboardFlag = keyboardTest
if vals.onKeyboardFlag:
vals.fingerBuff[0].put(fingerX)
vals.fingerBuff[1].put(fingerY)
# Move the mouse
if vals.fingerBuff[0].size() >= 2:
mouseX, mouseY = finger2MouseRelative(vals.fingerBuff[0].data, \
vals.fingerBuff[1].data, vals.traceX, vals.traceY)
vals.buff[0].put(mouseX)
vals.buff[1].put(mouseY)
smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))
# print fingerX, fingerY, mouseX, mouseY
vals.traceX, vals.traceY = smoothX, smoothY
if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
m.move(vals.traceX, vals.traceY)
# Absolute:
elif not vals.relativeFlag:
if not vals.zoom_flg:
vals.fX=rpt[tipIndex][0]
vals.fY=rpt[tipIndex][1]
if ((vals.inputX2-vals.inputX1)==0) or ((vals.inputY2-vals.inputY1)==0):
# Use default option
mouseX, mouseY = finger2Mouse(fingerX, fingerY, False)
# mouseX=(rpt[tipIndex][0]-600)*vals.width/vals.windowX
# mouseY=(rpt[tipIndex][1]-150)*vals.height/vals.windowY
else:
# Use the user prefered window size
mouseX, mouseY = finger2Mouse(fingerX, fingerY, True)
if not vals.zoom_flg:
vals.fingerX=fingerX
vals.fingerY=fingerY
vals.mouseX=mouseX
vals.mouseY=mouseY
"""Currently we have the setting such that if there is a single LED that is out of range then
the mouse wont move. The problem with this is that the range of the mouse gets limited, and
some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice
versa. So this is a give or take until we have a better filtering method."""
if (vals.inrange and doDepth.checkIndexInBox()) or vals.mouseState == vals.MOUSE_DRAG:
vals.buff[0].put(mouseX)
vals.buff[1].put(mouseY)
smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))
if vals.featureFlag:
vals.constBuff[0].put(mouseX)
vals.constBuff[1].put(mouseY)
sX = np.mean(fun.smooth(vals.constBuff[0].data, window_len = vals.constBuff[0].size()))
sY = np.mean(fun.smooth(vals.constBuff[1].data, window_len = vals.constBuff[1].size()))
# The speed of the cursor
speed = np.sqrt( (sX - vals.traceX)**2 + (sY - vals.traceY)**2 )
if speed < 0.0001:
speed = 0.0001
vals.speedBuff.put(speed)
vals.smoothSpeed = np.mean(fun.smooth(vals.speedBuff.data, window_len = vals.speedBuff.size()))
# Several method to get buffer size from speed:
# paramA, paramB = 8.5, 20
# 1) size = A + B / speed
# newSize = max(int(paramA + paramB / vals.smoothSpeed), vals.minBuffSize)
# 2) size = A + B / sqrt(speed)
# newSize = max(int(paramA + paramB / np.sqrt(vals.smoothSpeed)), vals.minBuffSize)
# newSize = min(newSize, vals.maxBuffSize)
# P1(minSpeed, maxBuff), P2(maxSpeed, minBuff)
maxSpeed = 25
minSpeed = 0.1
maxBuff = 35
minBuff = 10
# 3) size = A + B * speed
paramB = float(minBuff - maxBuff) / (maxSpeed - minSpeed)
paramA = maxBuff - paramB * minSpeed
newSize = paramA + paramB * vals.smoothSpeed
newSize = max(min(int(newSize), maxBuff), minBuff)
vals.buff[0].setBuffSize(newSize)
vals.buff[1].setBuffSize(newSize)
if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
# Record the last trace point
vals.traceX, vals.traceY = smoothX, smoothY
# if vals.featureFlag:# and vals.mouseState == vals.MOUSE_READY:
# # param = 20.0 / speed2
# # vals.traceX = int((vals.traceX * param + smoothX) / (1 + param))
# # vals.traceY = int((vals.traceY * param + smoothY) / (1 + param))
# vals.traceX = np.mean(fun.smooth(vals.buff[0].data[-vals.smoothSize:], window_len = vals.smoothSize))
# vals.traceY = np.mean(fun.smooth(vals.buff[1].data[-vals.smoothSize:], window_len = vals.smoothSize))
# # vals.traceX = (vals.traceX * 4 + smoothX) / 5
# # vals.traceY = (vals.traceY * 4 + smoothY) / 5
m.move(vals.traceX, vals.traceY)
# m.move(vals.buff[0].data[-1],vals.buff[1].data[-1])
# m.move(smoothX, smoothY)
# m.move(mouseX, mouseY)
if vals.wiimoteNum >= vals.wiimoteMaxNum \
and not (vals.calibLoadFlag or vals.calibration or vals.rec_flg):
doDraw.drawDefault(screen, defaultFont)
eventsObject=pygame.event.get()
doEvents.eventHandling(eventsObject)
msElapsed=clock.tick(pygameRate)
pygame.display.update()
if kill==True or vals.quit_FLG:
pygame.quit()
sys.exit()
break
for event in eventsObject:
if (event.type == QUIT) or (event.type == KEYDOWN and event.key == K_ESCAPE):
kill=True
pygame.quit()
sys.exit()
break
def drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2,tipIndex2, tipThumb2,kThumb2,kIndex2,averageX,averageY,myfont,calibFont,depthFont):
mouseModeDistance=fun.distanceVec(\
[rpt[tipIndex][0]],\
[rpt[tipIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
clickingDistance=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
screen.fill(vals.black)
#Drawing the Circles
pygame.draw.circle(screen, vals.yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
pygame.draw.circle(screen, vals.red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
pygame.draw.circle(screen, vals.green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
pygame.draw.circle(screen, vals.blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
pygame.draw.circle(screen, vals.white, (averageX/3,averageY/3),10)
#Drawing the instructions
pygame.draw.rect(screen, vals.gray, (0,5,600,60))
if vals.calibState == vals.START_CALIB:
Calib1=calibFont.render("Press H to calibrate the switching gesture",1,vals.black)
screen.blit(Calib1,(0,15))
elif vals.calibState == vals.MOUSE_MODE_CALIB:
Calib1=calibFont.render("Tap tip of thumb and tip of index",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),5 )
vals.mouseModeCalibList.append(mouseModeDistance[0])
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
vals.boxBoundCalibList.append(doDepth.meanDepth())
# Use 3d-box instead just using z-axis information
if vals.depthBuff[2].size() == 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# log the depth and index tip raw coordinate
vals.switchBoxData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
elif vals.calibState == vals.READY_CLICK_CALIB:
Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to calibrate the clicking gesture",1,vals.black)
screen.blit(Calib2,(0,35))
elif vals.calibState == vals.CLICK_CALIB:
Calib1=calibFont.render("Tap tip of thumb and knuckle of index for {} times".format(\
str(vals.clickNum)), 1, vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[kIndex][0]/3,rpt[kIndex][1]/3),5 )
vals.clickingCalibList[0].append(time.time() - vals.clickCalibSTime)
vals.clickingCalibList[1].append(clickingDistance[0])
elif vals.calibState == vals.READY_DEPTH_CALIB:
Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to calibrate keyboard plane",1,vals.black)
screen.blit(Calib2,(0,35))
elif vals.calibState == vals.DEPTH_CALIB:
Calib1=calibFont.render("Draw 5 circles on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
# Show the depth of the four LED : 440, 460, 480, 500
ledDepthKey = ['tipThumb: ', 'knuThumb: ', 'tipIndex: ', 'knuIndex: ']
for i in range(len(ledDepthKey)):
ledDepth = calibFont.render(ledDepthKey[i] + str(vals.depthBuff[i].back()), 1, vals.white)
screen.blit(ledDepth, (0, 80 + i * 20))
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
if vals.depthBuff[2].size() == 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# log the depth and index tip raw coordinate
vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
elif vals.calibState == vals.END_CALIB:
calibrationDone=1
Calib1=calibFont.render("Calibration Completed",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press r to start recording",1,vals.black)
screen.blit(Calib2,(0,35))
def drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,averageX2,averageY2,myfont, calibFont,depthFont):
# screen.fill(vals.black)
if not vals.testTypeFlag:
# Normal Recording Mode
mouseLabel=myfont.render("Mouse:"+" "+str(vals.mouseModeValue) ,1,(255,255,255))
screen.blit(mouseLabel,(0,80))
clickLabel=myfont.render("Click:"+" "+str(vals.clickValue) ,1,(255,255,255))
screen.blit(clickLabel,(0,95))
Calib1=calibFont.render("tipThumb:"+str(int(vals.depthBuff[0].mean())),1,vals.white)
screen.blit(Calib1,(0,115))
Calib2=calibFont.render("kThumb:"+str(int(vals.depthBuff[1].mean())),1,vals.white)
screen.blit(Calib2,(0,135))
Calib3=calibFont.render("tipIndex:"+str(int(vals.depthBuff[2].mean())),1,vals.white)
screen.blit(Calib3,(0,155))
Calib4=calibFont.render("kIndex:"+str(int(vals.depthBuff[3].mean())),1,vals.white)
screen.blit(Calib4,(0,175))
#tipDistance=calibFont.render("Switch-Distance:"+str(int(vals.tipDistance)),1,vals.white)
#screen.blit(tipDistance,(0,205))
#clickDistance=calibFont.render("Click-Distance:"+str(int(vals.clickDistance)),1,vals.white)
#screen.blit(clickDistance,(0,235))
x,y = rpt[tipIndex][0], rpt[tipIndex][1]
z = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
distance = getPlaneDistance(vals.planeParam, x, y, z)
keyboardTop = vals.planeParam[-1]
tipDistance=calibFont.render("Distance:"+str((distance)),1,vals.white)
screen.blit(tipDistance,(0,205))
clickDistance=calibFont.render("Keyboard Top:"+str((keyboardTop)),1,vals.white)
screen.blit(clickDistance,(0,235))
distance3D=calibFont.render("3D-Distance:"+str(int(vals.dist3D)),1,vals.white)
screen.blit(distance3D,(0,255))
box3D=calibFont.render(str(int(vals.boxLimit)),1,vals.white)
screen.blit(box3D,(0,285))
#Circles to check on mode switching
# if doDepth.checkAllInBox(): # Used to log data
if vals.inSwitchBox:
pygame.draw.circle(screen, vals.green, (10,325),10)
else:
pygame.draw.circle(screen, vals.red, (10,325),10)
inBox=calibFont.render("in 3dBox",1,vals.white)
screen.blit(inBox,(20,325))
vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
if (vals.inrange==1):
pygame.draw.circle(screen, vals.green, (10,345),10)
else:
pygame.draw.circle(screen, vals.red, (10,345),10)
inBox=calibFont.render("inrange",1,vals.white)
screen.blit(inBox,(20,345))
if (not vals.mouseSwitched_flg): # Used to log data
pygame.draw.circle(screen, vals.green, (10,365),10)
else:
pygame.draw.circle(screen, vals.red, (10,365),10)
inBox=calibFont.render("open tips",1,vals.white)
screen.blit(inBox,(20,365))
# Show window size which is used to control the sensitivity
windowSize = calibFont.render('window size X:{} Y:{}'.format(str(vals.windowX), str(vals.windowY)), 1, vals.white)
screen.blit(windowSize, (0, 400))
# Show the buffer size which is modified by the speed in the new feature(shift+f to turn on/off)
if vals.featureFlag:
buffSize = calibFont.render('current buffer size: {}'.format(str(vals.buff[0].size())), 1, vals.white)
else:
buffSize = calibFont.render('default buffer size: {}'.format(str(vals.defaultBuffSize)), 1, vals.white)
screen.blit(buffSize, (0, 420))
# Show the depth of the four LED : 440, 460, 480, 500
ledDepthKey = ['tipThumb: ', 'knuThumb: ', 'tipIndex: ', 'knuIndex: ']
for i in range(len(ledDepthKey)):
ledDepth = calibFont.render(ledDepthKey[i] + str(vals.depthBuff[i].back()), 1, vals.white)
screen.blit(ledDepth, (0, 440 + i * 20))
if vals.relativeFlag:
controlMode = calibFont.render('Relative Mode', 1, vals.white)
screen.blit(controlMode, (0, 520))
onKeyboard = calibFont.render('Keyboard Test: ' + str(vals.onKeyboardFlag), 1, vals.white)
screen.blit(onKeyboard, (0, 540))
else:
controlMode = calibFont.render('Absolute Mode', 1, vals.white)
screen.blit(controlMode, (0, 520))
rawXY = calibFont.render('Raw X:{}, Y:{}'.format(rpt[tipIndex][0], rpt[tipIndex][1]), 1, vals.white)
screen.blit(rawXY, (0, 560))
traceXY = calibFont.render('Mouse X:{}, Y:{}'.format(int(vals.traceX), int(vals.traceY)), 1, vals.white)
screen.blit(traceXY, (0, 580))
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
distance = getPlaneDistance(vals.planeParam, rpt[tipIndex][0], rpt[tipIndex][1], smoothTipIndex)
tipIndexDepth = calibFont.render('Tip Index Depth: {}'.format(distance), 1, vals.white)
screen.blit(tipIndexDepth, (0, 600))
#main circles
pygame.draw.circle(screen, vals.red, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
pygame.draw.circle(screen, vals.blue, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
pygame.draw.circle(screen, vals.green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
pygame.draw.circle(screen, vals.white, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
pygame.draw.circle(screen, vals.gray, (averageX/3,averageY/3),13)
pygame.draw.circle(screen, vals.gray, (averageX2/3,averageY2/3),13)
pygame.draw.circle(screen, vals.red, (rpt2[tipIndex2][0]/3,rpt2[tipIndex2][1]/3),10)
pygame.draw.circle(screen, vals.blue, (rpt2[kIndex2][0]/3,rpt2[kIndex2][1]/3),10)
pygame.draw.circle(screen, vals.green, (rpt2[tipThumb2][0]/3,rpt2[tipThumb2][1]/3),10)
pygame.draw.circle(screen, vals.white, (rpt2[kThumb2][0]/3,rpt2[kThumb2][1]/3),10)
#GUI for depth
# pygame.draw.rect(screen, vals.gray, (500,0,1500,1500))
depthGUILeft = int(vals.width * 0.35)
depthGUITop = int(vals.height * 0.05)
depthGUIWidth = int(vals.width * 0.45) - depthGUILeft
depthGUIHeight = int(vals.height * 0.95) - depthGUITop
pygame.draw.rect(screen, vals.gray, (depthGUILeft, depthGUITop, depthGUIWidth, depthGUIHeight))
# 0left--1startPos--2green--3white--4red--5blue--6endPos--7label--8right
objXPos = np.linspace(depthGUILeft, depthGUILeft + depthGUIWidth, 9)
objXPos = [int(x) for x in objXPos]
#Creating the lines
for i in xrange(11):
offsetY = 75
startPos = (objXPos[1], offsetY + i*30) # was 550
endPos = (objXPos[6], offsetY + i*30) # was 650
pygame.draw.line(screen, vals.black, startPos, endPos)
depthLabel = depthFont.render( str(5*i), 1, vals.black)
screen.blit(depthLabel, (objXPos[7], offsetY + i*30))
#Depth circles. Was 560, 580, 600, 620
pygame.draw.circle(screen, vals.green, (objXPos[2], int(75 + vals.depthBuff[0].mean()*6)), 10) #tipThumb
pygame.draw.circle(screen, vals.white, (objXPos[3], int(75 + vals.depthBuff[1].mean()*6)), 10) #kThumb
pygame.draw.circle(screen, vals.red, (objXPos[4], int(75 + vals.depthBuff[2].mean()*6)), 10) #tipindex
pygame.draw.circle(screen, vals.blue, (objXPos[5], int(75 + vals.depthBuff[3].mean()*6)), 10)#kIndex
#The gesture bounds
pygame.draw.line(screen,vals.red, (vals.gestureLeftThreshHold/3,0),(vals.gestureLeftThreshHold/3,800))
pygame.draw.line(screen,vals.blue, (0,vals.gestureDownThreshHold/3),(10000,vals.gestureDownThreshHold/3))
pygame.draw.line(screen,vals.yellow, (0,vals.gestureUpThreshHold/3),(10000,vals.gestureUpThreshHold/3))
#Mouses mode drawing
if vals.mouse_flg:
MouseKeyboard=myfont.render( "Mouse mode",1,(255,255,255))
else:
MouseKeyboard=myfont.render( "Keyboard mode",1,(255,255,255))
screen.blit(MouseKeyboard,(0,50))
#input rectangle
pygame.draw.line(screen,vals.white, (vals.inputX1/3,vals.inputY1/3), (vals.inputX2/3,vals.inputY1/3))
pygame.draw.line(screen,vals.white, (vals.inputX2/3,vals.inputY1/3), (vals.inputX2/3,vals.inputY2/3))
pygame.draw.line(screen,vals.white, (vals.inputX2/3,vals.inputY2/3), (vals.inputX1/3,vals.inputY2/3))
pygame.draw.line(screen,vals.white, (vals.inputX1/3,vals.inputY2/3), (vals.inputX1/3,vals.inputY1/3))
else:
# Testing Recording Mode: Show article text.
# I canceled the text input area.
testFont = 'textFiles/MonospaceTypewriter.ttf'
# Text GUI on the left: show the text
if vals.textContent == '':
# Read from file
try:
tcf = open(vals.textContentFile, 'r')
vals.textContent = tcf.read()
tcf.close()
print vals.textContent
except:
print 'Error: Invalid type content file.'
textGUILeft = int(vals.width * 0.05)
textGUITop = int(vals.height * 0.05)
textGUIWidth = int(vals.width * 0.45) - textGUILeft
textGUIHeight = int(vals.height * 0.95) - textGUITop
pygame.draw.rect(screen, vals.white, (textGUILeft, textGUITop, textGUIWidth, textGUIHeight))
if vals.textGUI == None:
textFontSize = 20
textBorder = 10
vals.textGUI = Reader(unicode(vals.textContent.expandtabs(4), 'utf8'), (textGUILeft + textBorder, textGUITop + textBorder), \
textGUIWidth - 2 * textBorder, textFontSize, height = textGUIHeight - 2 * textBorder, \
font = testFont, fgcolor = (0, 0, 0), hlcolor = (255,10,150,100), split = True)
vals.textGUI.show()
'''
def mouseActivities(pygame, rpt, tipIndex,tipThumb,kIndex,kThumb,m,k):
#3D Distance from the tipIndex to tipThumb
dist3D=euclidean(rpt[tipIndex],rpt[tipThumb])
vals.dist3D=dist3D
#Distance for switching modes
dista=fun.distanceVec(\
[rpt[tipIndex][0]],\
[rpt[tipIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
vals.tipDistance=dista[0]
#Distance for clicking - thumb tip to index knuckle
distClick=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
vals.clickDistance=distClick[0]
#Modifying vals.mouseModeValue with respect to distance between knuckles
# currentKnuckleValue=fun.distanceVec(\
# [rpt[kIndex][0]],\
# [rpt[kIndex][1]],\
# [rpt[kThumb][0]],\
# [rpt[kThumb][1]])[0]
# knuckleRatio=float(currentKnuckleValue/vals.knuckleValue)
# if knuckleRatio>1:
# newMouseModeValue=int(knuckleRatio*vals.mouseModeValue)
# newClickValue=int(knuckleRatio*vals.clickValue)
# else:
newMouseModeValue=vals.mouseModeValue
newClickValue=vals.clickValue
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
checkSwitchBox(rpt[tipIndex][0], rpt[tipIndex][1], smoothTipIndex)
# inBox = doDepth.checkAllInBox() # Used to log data
if vals.inSwitchBox:
#Switching Modes
#When distance tips goes below mouseModevalue, start measuring time.
if 10<=dista[0]<=newMouseModeValue and vals.inrange==1 and vals.mouseModeSwitchTime==0:
vals.mouseModeSwitchTime=time.time()
hold5ms = (vals.timeHold<=(time.time()-vals.mouseModeSwitchTime)*1000)
tipInRange= (10<=dista[0]<=newMouseModeValue)
mouseCondition= hold5ms and tipInRange# and vals.inrange==1
#if distance is below for a certain time and all other conditions are met, then switch
if mouseCondition and vals.mouse_flg==0 and not vals.mouseSwitched_flg:
print('Mouse mode activated')
vals.traceX, vals.traceY = m.position()
vals.mouse_flg=1
vals.mouseModeSwitchTime=0
vals.mouseSwitched_flg=1
try:
vals.switchSound.play()
except:
pass
# file = 'switch.mp3'
# try:
# vals.a.play()
# except:
# vals.a.load(file)
# vals.a.play()
if mouseCondition and vals.mouse_flg==1 and not vals.mouseSwitched_flg:
print('Mouse mode deactivated')
vals.mouse_flg=0
vals.contDist=0
vals.mouseSwitched_flg=1
try:
vals.switchSound.play()
except:
pass
# file = 'switch.mp3'
# try:
# vals.a.play()
# except:
# vals.a.load(file)
# vals.a.play()
#after switching, the fingers need to part in order to reset constants.
if (vals.mouseSwitched_flg and dista[0]>newMouseModeValue):
vals.mouseSwitched_flg=0
vals.mouseModeSwitchTime=0
'''
#Adjusting MaxBuff with respect to thumbtip and index knuckle
# It doesn't work. Currently this method uses a larger buffer when clicking,
# in order to avoid mistakes when getting the clickX and clickY.
# But this method will only work when we put the cursor still on the target.
# If we move and press 'click', it will be worse, because of its larger buffer contains
# more wrong information.
# Method 1: Sin
# if vals.mouse_flg and vals.debugFlag:
# currBuff = vals.minBuff
# if distClick[0] < 0.8 * newClickValue:
# currBuff = vals.maxBuff
# elif distClick[0] > 1.2 * newClickValue:
# currBuff = vals.minBuff
# else:
# currBuff = (vals.maxBuff + vals.minBuff)/2 - (vals.maxBuff - vals.minBuff)/2 * \
# np.sin((distClick[0] - newClickValue) * 2 * np.pi / 0.8 / newClickValue)
# vals.buff[0].setCurrBuff(currBuff)
# vals.buff[1].setCurrBuff(currBuff)
# a=40*newClickValue
# vals.maxBuff=a/distClick[0]
# if vals.maxBuff<20:
# vals.maxBuff=20
# elif vals.maxBuff>40:
# vals.maxBuff=40
'''
# Clicking and Dragging
if vals.mouseState == vals.MOUSE_NORMAL:
# print 'NORMAL'
#print distClick[0], vals.inrange, vals.mouse_flg
if distClick[0] <= newClickValue and vals.inrange and vals.mouse_flg:
# Get possible point of click or drag
vals.clickX, vals.clickY = vals.traceX, vals.traceY
# vals.clickX = np.mean(fun.smooth(vals.buff[0].data, window_len=len(vals.buff[0].data)))
# vals.clickY = np.mean(fun.smooth(vals.buff[1].data, window_len=len(vals.buff[1].data)))
vals.dragX, vals.dragY = vals.clickX, vals.clickY
vals.stime = time.time()
vals.mouseState = vals.MOUSE_READY
#vals.mouseActBuff = [[], []]
print 'READY'
print 'distClick[0]: ' + str(distClick[0])
elif vals.mouseState == vals.MOUSE_READY:
# print 'READY'
currTime = (time.time() - vals.stime) * float(1000)
# Note: We don't need to add time threshold here to do READY -> CLICK.
# Because the point is discrete, it may jump from the READY rectangle to NORMAL without
# triggering CLICK or DRAG signal. That is, its distClick < clickValue and time < timeThre for
# the last point, but distClick > clickValue and time > timeThre for the current point.
# In this case, we assume that is a CLICK. Noted by Zhen Li, Aug 5th, 2014.
if distClick[0] > newClickValue and vals.mouse_flg and vals.inrange:# and currTime <= vals.mouseActTimeThre:
# Click
vals.mouseState = vals.MOUSE_CLICK
# if not vals.testTypeFlag:
# m.click(vals.clickX, vals.clickY)
# Detect double click:
clickTime = (time.time() - vals.lastClickTime) * float(1000)
if clickTime > vals.doubleClickTimeThre or not vals.dragFlag:
# Click
if not vals.testTypeFlag:
m.click(vals.clickX, vals.clickY)
vals.lastClickX, vals.lastClickY = vals.clickX, vals.clickY
print('Click')
try:
vals.clickSound.play()
except:
pass
# file = 'click.mp3'
# try:
# vals.b.play()
# except:
# vals.b.load(file)
# vals.b.play()
else:
# Double Click
if not vals.testTypeFlag:
m.click(vals.lastClickX, vals.lastClickY)
print('Double Click')
try:
vals.clickSound.play()
except:
pass
# file = 'click.mp3'
# try:
# vals.b.play()
# except:
# vals.b.load(file)
# vals.b.play()
vals.lastClickTime = time.time()
# print('Click')
print 'distClick[0]: ' + str(distClick[0])
elif distClick[0] <= newClickValue and vals.mouse_flg and vals.inrange and currTime > vals.mouseActTimeThre:
# Drag if enabled
if not vals.testTypeFlag:
if vals.dragFlag:
m.press(vals.dragX, vals.dragY)
else:
m.click(vals.clickX, vals.clickY)
try:
vals.clickSound.play()
except:
pass
vals.mouseState = vals.MOUSE_DRAG
print('Drag')
print 'distClick[0]: ' + str(distClick[0])
elif vals.mouseState == vals.MOUSE_CLICK:
# print 'CLICK'
# vals.mouseState = vals.MOUSE_NORMAL
vals.mouseState = vals.MOUSE_WAIT
elif vals.mouseState == vals.MOUSE_WAIT:
if distClick[0] > 1.5 * newClickValue and vals.mouse_flg and vals.inrange:
vals.mouseState = vals.MOUSE_NORMAL
elif vals.mouseState == vals.MOUSE_DRAG:
# print 'DRAG'
if distClick[0] > newClickValue and vals.mouse_flg:
# Release if enabled
if vals.dragFlag and not vals.testTypeFlag:
m.release(vals.buff[0].mean(),vals.buff[1].mean())
vals.mouseState = vals.MOUSE_NORMAL
print("Release")
print 'distClick[0]: ' + str(distClick[0])
if vals.testTypeFlag or vals.testPointFlag:
''' TODO: convert them to actual coordinate on the display.'''
tIX, tIY = finger2Mouse(rpt[tipIndex][0], rpt[tipIndex][1])
kIX, kIY = finger2Mouse(rpt[kIndex][0], rpt[kIndex][1])
tTX, tTY = finger2Mouse(rpt[tipThumb][0], rpt[tipThumb][1])
kTX, kTY = finger2Mouse(rpt[kThumb][0], rpt[kThumb][1])
smoothX, smoothY = vals.traceX, vals.traceY
# smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len=len(vals.buff[0].data)))
# smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len=len(vals.buff[1].data)))
# time, dista0, distClick0, inrange, inBox
# tIX, tIY, kIX, kIY, tTX, tTY, kTX, kTY, smoothX, smoothY
# mouse_flg, mouseState, clickX, clickY, speed, buffSize
vals.testTypeData.append('{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}'.format(\
str(time.time() - vals.testStartTime), str(dista[0]), str(distClick[0]), str(int(vals.inrange)), str(int(vals.inSwitchBox)), \
str(tIX), str(tIY), str(kIX), str(kIY), str(tTX), str(tTY), str(kTX), str(kTY), str(smoothX), str(smoothY), \
str(vals.mouse_flg), str(vals.mouseState), str(vals.clickX), str(vals.clickY), str(vals.smoothSpeed), str(vals.buff[0].size())
))
def getDistAndTime(X, Y, paramD, paramT):
"Get mean and std of distance and time(ms) with calib data: meanDist, stdDist, meanTime, stdTime"
"Then get answer = mean + std * param"
# plot(X, Y, 'g.')
windowLen = 16
smoothY = smooth(Y, windowLen)
smoothY = smoothY[(windowLen / 2 - 1) : -(windowLen / 2)]
# plot(X, smoothY, 'c-')
# Note: The author suggests: '(sample / period) / f' where '4 >= f >= 1.25'
f = 6
lookahead = len(X) / vals.clickNum / f
_max, _min = peakdetect(smoothY, X, lookahead)#, 0.30)
# print _max
# print _min
xm = [p[0] for p in _max]
ym = [p[1] for p in _max]
xn = [p[0] for p in _min]
yn = [p[1] for p in _min]
# plot(xm, ym, 'rx', markersize = 10)
# plot(xn, yn, 'kx', markersize = 10)
yTh = getYTh(ym, yn)
# print 'yTh: ' + str(yTh)
newXMax = []
newYMax = []
newXMin = []
newYMin = []
i, j = 0, 0
lastPoint = 0
ADD_MAX, \
ADD_MIN = range(2)
if xm[i] < xn[j]:
#max,min,...
newXMax.append(xm[i])
newYMax.append(ym[i])
i = i + 1
lastPoint = ADD_MAX
else:
#min, max,... : add a max point
newXMax.append(X[0])
newYMax.append(smoothY[0])
lastPoint = ADD_MAX
while i < len(_max) and j < len(_min):
if lastPoint == ADD_MAX:
if xm[i] > xn[j]:
# print 'max, [min], max: ' + str(ym[i-1] - yn[j])
if (ym[i-1] - yn[j]) > yTh:
#...max, [min], max... : Normal
# print 'add min:{}, x={}'.format(str(j), xn[j])
newXMin.append(xn[j])
newYMin.append(yn[j])
j = j + 1
lastPoint = ADD_MIN
#Special judge for wrong [max] before it
try:
if i >= 1 and newXMax[-1] == X[0] and newXMax[-1] < xm[i-1]:
#print newXMax[-1], xm[i-1], i
newXMax[-1] = xm[i-1]
newYMax[-1] = ym[i-1]
except:
pass
else:
#Discard the [min]
j = j + 1
else:
#...max, [max], min...: Discard it
i = i + 1
elif lastPoint == ADD_MIN:
if xm[i] < xn[j]:
# print 'min, [max], min: ' + str(ym[i] - yn[j -1])
if (ym[i] - yn[j -1]) > yTh:
#...min, [max], min...: Normal
# print 'add max:{}, x={}'.format(str(i), xm[i])
newXMax.append(xm[i])
newYMax.append(ym[i])
i = i + 1
lastPoint = ADD_MAX
else:
#Discard the [max]
i = i + 1
else:
#...min, [min], max: Discard it
# newXMax.append((xn[j] + xn[j-1]) / 2)
# newYMax.append(ym[0])
# newXMin.append(xn[j])
# newYMin.append(yn[j])
j = j + 1
# lastPoint = ADD_MIN
else:
if i < len(_max) and lastPoint == ADD_MIN:
#...min, [max].
newXMax.append(xm[i])
newYMax.append(ym[i])
i = i + 1
lastPoint = ADD_MAX
elif j < len(_min) and lastPoint == ADD_MAX and (ym[i-1] - yn[j]) > yTh:
#...max, [min] : Make a max after it
newXMin.append(xn[j])
newYMin.append(yn[j])
newXMax.append(X[-1])
newYMax.append(smoothY[-1])
j = j + 1
lastPoint = ADD_MAX
# plot(newXMax, newYMax, 'r*', markersize = 10)
# plot(newXMin, newYMin, 'k*', markersize = 10)
# print fi
meanDist = np.mean(newYMin)
stdDist = np.std(newYMin)
targetDist = meanDist + stdDist * paramD
print 'Dist: mean:{}, std:{}'.format(str(meanDist), str(stdDist))
clickTimes = getClickTimes(X, smoothY, newXMin, targetDist)
meanTime = np.mean(clickTimes)
stdTime = np.std(clickTimes)
targetTime = meanTime + stdTime * paramT
print 'Time: mean:{}, std:{}'.format(str(meanTime), str(stdTime))
return targetDist, targetTime * 1000
def capture(self):
#Initialization of parameters
contDist=0
inrange=0
red = (255,0,0,120)
green = (0,255,0)
blue = (0,0,255)
darkBlue = (0,0,128)
white = (255,255,255)
black = (0,0,0)
yellow = (255,255,0)
gray= (205,200, 177)
rpt=[ [0,0] for i in range(4)]
X=0
Y=0
t=[]
ir_x=[]
ir_y=[]
ir_s=[]
t_i = time.time()
m = PyMouse()
k = PyKeyboard()
running=False
buff=[[],[]]
maxBuff=20
buff[0]=q.miniQueue(maxBuff)
buff[1]=q.miniQueue(maxBuff)
gestures=[[6,2,8,2,8],[9,3,9]]
gestures=[ [str(i2) for i2 in i]for i in gestures]
#recording flags
rec_flg =0
flg=True
mouse_flg=0
click_flg=0
doubleClick_flg=0
drag_flg=0
dragX=0
dragY=0
wait_flg=0
timeHold=80 #in milliseconds
#calibration
mouseModeValue=80
clickValue=80
lagValue=100
calibration=False
mouseModeCalib=False
startMouseModeCalib=False
clickingCalib=False
startClickModeCalib=False
mouseModeCalibList=[]
clickingCalibList=[]
rightClickValue=180
#Gesture
gestureRightThreshHold=1000
gestureLeftThreshHold=300
gestureDownThreshHold=720
gestureUpThreshHold=400
gesture_flg_UD=0
gesture_flg_DU=0
gesture_flg_LR=0
gesture_flg_RL=0
#Check inrange
LED1=[]
LED2=[]
LED3=[]
LED4=[]
rptList=[]
#Intialization of GUI
os.environ['SDL_VIDEO_iWINDOW_POS'] = "%d,%d" % (0,0)
pygame.init()
myfont=pygame.font.SysFont("monospace",15)
calibFont=pygame.font.SysFont(",monospace",20)
#the default cursor
DEFAULT_CURSOR = mouse.get_cursor()
#the hand cursor
_HAND_CURSOR = (
" XX ",
" X..X ",
" X..X ",
" X..X ",
" X..XXXXX ",
" X..X..X.XX ",
" XX X..X..X.X.X ",
"X..XX.........X ",
"X...X.........X ",
" X.....X.X.X..X ",
" X....X.X.X..X ",
" X....X.X.X.X ",
" X...X.X.X.X ",
" X.......X ",
" X....X.X ",
" XXXXX XX ")
_HCURS, _HMASK = pygame.cursors.compile(_HAND_CURSOR, ".", "X")
HAND_CURSOR = ((16, 16), (5, 1), _HCURS, _HMASK)
infoObject = pygame.display.Info()
width=infoObject.current_w
height=infoObject.current_h
screen=pygame.display.set_mode((width/2,height/2))
#screen=pygame.display.set_mode((1200/3,760/3))
print('press "c" to calibrate, then')
print('press "r" to start recording')
while flg==True:
if calibration: #do calibration
newList=self.findDegrees(rpt) #[(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=self.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=self.thumbData(newList)
averageX,averageY=self.centerFind(rpt)
#GUI section
screen.fill(black)
#Drawing the Circles
pygame.draw.circle(screen, yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
pygame.draw.circle(screen, red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
pygame.draw.circle(screen, green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
pygame.draw.circle(screen, blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
pygame.draw.circle(screen, white, (averageX/3,averageY/3),10)
mouseModeDistance=fun.distanceVec(\
[rpt[tipIndex][0]],\
[rpt[tipIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
clickingDistance=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
pygame.draw.rect(screen, gray, (0,5,500,60))
if not (mouseModeCalib or startClickModeCalib or startMouseModeCalib or clickingCalib):
Calib1=calibFont.render("Press H to start",1,black)
screen.blit(Calib1,(0,15))
if startMouseModeCalib and not mouseModeCalib:
Calib1=calibFont.render("Tap tip of thumb and tip of index",1,black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),5 )
mouseModeCalibList.append(mouseModeDistance[0])
if startClickModeCalib and not clickingCalib:
Calib1=calibFont.render("Tap tip of thumb and knuckle of index",1,black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[kIndex][0]/3,rpt[kIndex][1]/3),5 )
clickingCalibList.append(clickingDistance[0])
if mouseModeCalib and clickingCalib:
calibrationDone=1
Calib1=calibFont.render("Calibration Completed",1,black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press r to start recording",1,black)
screen.blit(Calib2,(0,35))
#Recording
if rec_flg==1:
newList=self.findDegrees(rpt) #[(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=self.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=self.thumbData(newList)
averageX,averageY=self.centerFind(rpt)
#GUI section
screen.fill(black)
#Drawing the Circles
pygame.draw.circle(screen, yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
pygame.draw.circle(screen, red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
pygame.draw.circle(screen, green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
pygame.draw.circle(screen, blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
pygame.draw.circle(screen, white, (averageX/3,averageY/3),10)
#Drawing the Information Text
ITLabel=myfont.render( "IndexTip"+" "+str(tipIndexAngle),1,(25,255,255))
screen.blit(ITLabel,(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3))
IKLabel=myfont.render( "IndexKnuck"+" "+str(kIndexAngle) ,1,(255,255,255))
screen.blit(IKLabel,(rpt[kIndex][0]/3,rpt[kIndex][1]/3))
TTLabel=myfont.render( "ThumbTip"+" "+str(tipThumbAngle) ,1,(255,255,255))
screen.blit(TTLabel,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3))
TKLabel=myfont.render( "ThumbKnuck"+" "+str(kThumbAngle) ,1,(255,255,255))
screen.blit(TKLabel,(rpt[kThumb][0]/3,rpt[kThumb][1]/3))
speedLabel=myfont.render("Increase:z, Decrease:x",1,(255,255,255))
screen.blit(speedLabel,(0,65))
mouseLabel=myfont.render("Mouse:"+" "+str(mouseModeValue) ,1,(255,255,255))
screen.blit(mouseLabel,(0,80))
clickLabel=myfont.render("Click:"+" "+str(clickValue) ,1,(255,255,255))
screen.blit(clickLabel,(0,95))
#Mouse Events
#Drawing the mode
if mouse_flg:
MouseKeyboard=myfont.render( "Mouse mode",1,(255,255,255))
else:
MouseKeyboard=myfont.render( "Keyboard mode",1,(255,255,255))
screen.blit(MouseKeyboard,(0,50))
#Distance for switching modes
dista=fun.distanceVec(\
[rpt[tipIndex][0]],\
[rpt[tipIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
#Distance for clicking - thumb tip to index knuckle
distClick=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
#Switching Modes
if 10<=dista[0]<=mouseModeValue and inrange==1:
contDist+=1
if contDist>=timeHold and mouse_flg==0 and drag_flg==0:
print('Mouse mode activated')
mouse_flg=1
contDist=0
mouse.set_cursor(*HAND_CURSOR)
if contDist>=timeHold and mouse_flg==1 and drag_flg==0:
print('Mouse mode deactivated')
mouse_flg=0
contDist=0
mouse.set_cursor(*DEFAULT_CURSOR)
#Adjusting MaxBuff with respect to thumbtip and index knuckle
if mouse_flg:
a=40*clickValue
maxBuff=a/distClick[0]
if maxBuff<20:
maxBuff=20
elif maxBuff>40:
maxBuff=40
#Clicking
if distClick[0]<clickValue and inrange and mouse_flg and not click_flg:
click_flg=1
stime=time.time()
m.click(buff[0].mean(),buff[1].mean())
dragX, dragY=buff[0].mean(),buff[1].mean()
print('Click')
print distClick[0]
if (click_flg and (time.time()-stime)*1000>=lagValue and not drag_flg): #so its been 1/2 second,
if (distClick[0]>=clickValue): #if finger is up, then delete flag. Else
click_flg=0
drag_flg=0
print("reset")
print distClick[0]
elif ((dragX-buff[0].mean()>5) or (dragY-buff[1].mean()>5)): #Drag situation
m.press(dragX,dragY)
drag_flg=1
print ("dragging")
print distClick[0]
if drag_flg and distClick[0]>=int(1.2*clickValue): #released the drag
drag_flg=0
m.release(buff[0].mean(),buff[1].mean())
dragX,dragY=0,0
print("release drag")
print distClick[0]
#right click needs improvement #tried using the thumb
#if mouse_flg and rightClick[0]>rightClickValue:
# m.click(buff[0].mean(),buff[1].mean(),2)
#Gestures
#The gesture bounds
# pygame.draw.line(screen,white, (gestureRightThreshHold/3,0),(gestureRightThreshHold/3,800))
# pygame.draw.line(screen,red, (gestureLeftThreshHold/3,0),(gestureLeftThreshHold/3,800))
# pygame.draw.line(screen,blue, (0,gestureDownThreshHold/3),(10000,gestureDownThreshHold/3))
# pygame.draw.line(screen,yellow, (0,gestureUpThreshHold/3),(10000,gestureUpThreshHold/3))
#Swipe Right to Left
if self.allAboveGestureRight(rpt,gestureRightThreshHold) and not gesture_flg_RL:
gestureTime=time.time()
gesture_flg_RL=1
print("ready to gesture")
if gesture_flg_RL and (time.time()-gestureTime)<1:
if self.allAboveGestureLeft(rpt, gestureLeftThreshHold):
k.press_key(k.control_key)
k.press_key(k.alt_key)
k.press_key(k.left_key)
k.release_key(k.control_key)
k.release_key(k.alt_key)
k.release_key(k.left_key)
gesture_flg_RL=0
#Swipe Left to Right
if self.allAboveGestureLeft(rpt,gestureLeftThreshHold) and not gesture_flg_LR:
gestureTime=time.time()
gesture_flg_LR=1
print("ready to gesture")
if gesture_flg_LR and (time.time()-gestureTime)<1:
if self.allAboveGestureRight(rpt, gestureRightThreshHold):
k.press_key(k.control_key)
k.press_key(k.alt_key)
k.press_key(k.right_key)
k.release_key(k.control_key)
k.release_key(k.alt_key)
k.release_key(k.right_key)
gesture_flg_LR=0
#Swipe Down to Up
if self.allAboveGestureDown(rpt,gestureDownThreshHold) and not gesture_flg_DU:
gestureTime=time.time()
gesture_flg_DU=1
print("ready to gesture")
if gesture_flg_DU and (time.time()-gestureTime)<1:
if self.allAboveGestureUp(rpt, gestureUpThreshHold):
k.press_key(k.control_key)
k.press_key(k.alt_key)
k.press_key(k.up_key)
k.release_key(k.control_key)
k.release_key(k.alt_key)
k.release_key(k.up_key)
gesture_flg_DU=0
#Swipe Up to Down
if self.allAboveGestureUp(rpt,gestureUpThreshHold) and not gesture_flg_UD:
gestureTime=time.time()
gesture_flg_UD=1
print("ready to gesture")
if gesture_flg_UD and (time.time()-gestureTime)<1:
if self.allAboveGestureDown(rpt, gestureDownThreshHold):
k.press_key(k.control_key)
k.press_key(k.alt_key)
k.press_key(k.down_key)
k.release_key(k.control_key)
k.release_key(k.alt_key)
k.release_key(k.down_key)
gesture_flg_UD=0
if gesture_flg_RL and (time.time()-gestureTime)>=1:
gesture_flg_RL=0
if gesture_flg_LR and (time.time()-gestureTime)>=1:
gesture_flg_LR=0
if gesture_flg_UD and (time.time()-gestureTime)>=1:
gesture_flg_UD=0
if gesture_flg_DU and (time.time()-gestureTime)>=1:
gesture_flg_DU=0
#Capturing keyboard events
for event in pygame.event.get():
if event.type==KEYDOWN:
if event.key==pygame.K_r: #start recording
rec_flg=1
calibration=False
elif event.key==pygame.K_c: #start calibration
calibration=1
elif event.key==pygame.K_s: #pauses the recording
rec_flg=False
break
elif event.key==pygame.K_q: #quits entirely
flg=False
break
if rec_flg: #if recording, can change the lag time
if event.key==pygame.K_z:
lagValue+=100
elif event.key==pygame.K_x:
lagValue-=100
#Mouse events for calibration mode
if calibration:
if not mouseModeCalib:
if not startMouseModeCalib and event.key==pygame.K_h:
startMouseModeCalib=True
elif startMouseModeCalib and event.key==pygame.K_h:
mouseModeCalib=True
while min(mouseModeCalibList)<50:
mouseModeCalibList.remove(min(mouseModeCalibList))
mouseModeValue=int(1.2*min(mouseModeCalibList))
mouseModeCalib=True
if mouseModeCalib:
if not startClickModeCalib and event.key==pygame.K_h:
startClickModeCalib=True
elif startClickModeCalib and event.key==pygame.K_h:
while min(clickingCalibList)<30:
clickingCalibList.remove(min(clickingCalibList))
clickValue=int(1.2*min(clickingCalibList))
clickingCalib=True
if event.type==QUIT:
flg=False
pygame.quit()
break
#Capturing wii data
messages = self.wii.get_mesg()
for mesg in messages: # Loop through Wiimote Messages
if mesg[0] == cwiid.MESG_IR: # If message is IR data
#while recording data
if rec_flg == 1 or calibration: # If recording
cont=-1
for s in mesg[1]: # Loop through IR LED sources
cont+=1
if s: # If a source exists
t.append(time.time()-t_i)
rpt[cont][0]=(1200-s['pos'][0])
rpt[cont][1]=s['pos'][1]
#Check inrange
newRpt=copy.deepcopy(rpt)
rptList.append(newRpt)
inrange, LED1,LED2,LED3,LED4=self.rangeChecker(rptList, LED1, LED2,LED3,LED4)
#while in mouse mode
if mouse_flg==1:
X=rpt[tipIndex][0]
mouseX=(X-600)*width/400
Y=rpt[tipIndex][1]
mouseY=(Y-150)*height/290
"""Currently we have the setting such that if there is a single LED that is out of range then
the mouse wont move. The problem with this is that the range of the mouse gets limited, and
some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice
versa. So this is a give or take until we have a better filtering method."""
if inrange:
buff[0].put(mouseX)
buff[1].put(mouseY)
smoothX=np.mean(fun.smooth(buff[0].data, window_len=len(buff[0].data)))
smoothY=np.mean(fun.smooth(buff[1].data, window_len=len(buff[1].data)))
m.move(smoothX,smoothY)
#I can also control using the wiimote
elif mesg[0] == cwiid.MESG_BTN: # If Message is Button data
if mesg[1] & cwiid.BTN_PLUS: # Start Recording
rec_flg = 1
print "Recording..."
elif mesg[1] & cwiid.BTN_MINUS: # Stop Recording
flg=False
break
pygame.display.update()
pygame.quit()