def drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2,tipIndex2, tipThumb2,kThumb2,kIndex2,averageX,averageY,myfont,calibFont,depthFont):
mouseModeDistance=fun.distanceVec(\
[rpt[tipIndex][0]],\
[rpt[tipIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
clickingDistance=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
screen.fill(vals.black)
#Drawing the Circles
pygame.draw.circle(screen, vals.yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
pygame.draw.circle(screen, vals.red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
pygame.draw.circle(screen, vals.green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
pygame.draw.circle(screen, vals.blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
pygame.draw.circle(screen, vals.white, (averageX/3,averageY/3),10)
#Drawing the instructions
pygame.draw.rect(screen, vals.gray, (0,5,600,60))
if vals.calibState == vals.START_CALIB:
Calib1=calibFont.render("Press H to calibrate the switching gesture",1,vals.black)
screen.blit(Calib1,(0,15))
elif vals.calibState == vals.MOUSE_MODE_CALIB:
Calib1=calibFont.render("Tap tip of thumb and tip of index",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),5 )
vals.mouseModeCalibList.append(mouseModeDistance[0])
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
vals.boxBoundCalibList.append(doDepth.meanDepth())
# Use 3d-box instead just using z-axis information
if vals.depthBuff[2].size() == 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# log the depth and index tip raw coordinate
vals.switchBoxData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
elif vals.calibState == vals.READY_CLICK_CALIB:
Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to calibrate the clicking gesture",1,vals.black)
screen.blit(Calib2,(0,35))
elif vals.calibState == vals.CLICK_CALIB:
Calib1=calibFont.render("Tap tip of thumb and knuckle of index for {} times".format(\
str(vals.clickNum)), 1, vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[kIndex][0]/3,rpt[kIndex][1]/3),5 )
vals.clickingCalibList[0].append(time.time() - vals.clickCalibSTime)
vals.clickingCalibList[1].append(clickingDistance[0])
elif vals.calibState == vals.READY_DEPTH_CALIB:
Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to calibrate keyboard plane",1,vals.black)
screen.blit(Calib2,(0,35))
elif vals.calibState == vals.DEPTH_CALIB:
Calib1=calibFont.render("Draw 5 circles on the keyboard",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press H to complete",1,vals.black)
screen.blit(Calib2,(0,35))
# Show the depth of the four LED : 440, 460, 480, 500
ledDepthKey = ['tipThumb: ', 'knuThumb: ', 'tipIndex: ', 'knuIndex: ']
for i in range(len(ledDepthKey)):
ledDepth = calibFont.render(ledDepthKey[i] + str(vals.depthBuff[i].back()), 1, vals.white)
screen.blit(ledDepth, (0, 80 + i * 20))
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
if vals.depthBuff[2].size() == 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# log the depth and index tip raw coordinate
vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
elif vals.calibState == vals.END_CALIB:
calibrationDone=1
Calib1=calibFont.render("Calibration Completed",1,vals.black)
screen.blit(Calib1,(0,15))
Calib2=calibFont.render("Press r to start recording",1,vals.black)
screen.blit(Calib2,(0,35))
def run(self):
global FLG, coords
FLG = 1
m = PyMouse()
k = PyKeyboard()
#Intialization of Pygame
pygameRate=100
os.environ['SDL_VIDEO_iWINDOW_POS'] = "%d,%d" % (0,0)
pygame.mixer.pre_init(44100, -16, 2, 2048) # setup mixer to avoid sound lag
pygame.init()
vals.switchSound = pygame.mixer.Sound('switch.wav')
vals.clickSound = pygame.mixer.Sound('click.wav')
clock=pygame.time.Clock()
myfont=pygame.font.SysFont("monospace",15)
calibFont=pygame.font.SysFont("monospace",20)
depthFont=pygame.font.SysFont("monospace",10)
defaultFont=pygame.font.SysFont("monospace",15)
pygame.mixer.init()
infoObject = pygame.display.Info()
vals.width = infoObject.current_w
vals.height = infoObject.current_h
# print width, height
screen=pygame.display.set_mode((int(vals.width*0.5), vals.height))
global kill
#Main loop
while kill==False:
screen.fill((0,0,0))
rpt=[[int(i2) for i2 in i]for i in coords[0]]
rpt2=[[int(i2) for i2 in i]for i in coords[1]]
if vals.switchRPT:
tmp = rpt
rpt = rpt2
rpt2 = tmp
if not vals.rec_flg and (vals.calibration or vals.calibLoadFlag): #do calibration or load from file
#Receiving data from the threads
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
#GUI section
#doDraw.drawAllMiniCalibration(miniScreen, rpt, tipIndex, tipThumb,kThumb,kIndex,averageX,averageY,myfont,calibFont,depthFont)
doDraw.drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)
if vals.inputCalibration:
#Receiving data from the threads
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
#GUI section
doDraw.drawInputCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)
#first "click" get where the rpt index x,y, store it. Second click completes the line, turn into square
distClick=fun.distanceVec(\
[rpt[kIndex][0]],\
[rpt[kIndex][1]],\
[rpt[tipThumb][0]],\
[rpt[tipThumb][1]])
#first click
if (distClick[0]<vals.clickValue and vals.inputCounter==0):
vals.inputX1=rpt[tipIndex][0]
vals.inputY1=rpt[tipIndex][1]
vals.inputCounter+=1
vals.inputClickStopper=1
print "awef"
print vals.inputX1,vals.inputY1
if (distClick[0]>=vals.clickValue and vals.inputClickStopper):
vals.inputClickStopper=0
#second click
if (distClick[0]<vals.clickValue and vals.inputCounter==1 and not vals.inputClickStopper):
vals.inputX2=rpt[tipIndex][0]
vals.inputY2=rpt[tipIndex][1]
print "hey"
print vals.inputX2,vals.inputY2
vals.inputCounter+=1
vals.inputSet=1
if vals.inputSet:
#sets the inrange constants
if vals.inputX1<=vals.inputX2:
vals.leftBound=vals.inputX1
vals.rightBound=vals.inputX2
else:
vals.leftBound=vals.inputX2
vals.rightBound=vals.inputX1
if vals.inputY1<=vals.inputY2:
vals.upperBound=vals.inputY1
vals.lowerBound=vals.inputY2
else:
vals.upperBound=vals.inputY2
vals.lowerBound=vals.inputY1
vals.windowX = vals.rightBound - vals.leftBound
vals.windowY = vals.lowerBound - vals.upperBound
if vals.rec_flg==1: #Recording
#Finding out the location of the LEDs, tipThumb, kThumb....
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
averageX2,averageY2=findingPoints.centerFind(rpt2) #the center point
#Check whether LED is in range
newRpt=copy.deepcopy(rpt)
vals.rptList.append(newRpt)
vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
if (vals.inputX1<=rpt[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt[tipIndex][1]<=vals.inputY2) or \
(vals.inputX1<=rpt2[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt2[tipIndex][1]<=vals.inputY2):
vals.inrange=1
#Depth
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
#GUI
doDraw.drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,averageX2,averageY2,myfont,calibFont,depthFont)
#doDraw.drawAllMiniRecording(miniScreen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,myfont,calibFont,depthFont)
# Only log the depth data in record mode when testing or debugging
# if vals.relativeFlag:
# if vals.depthBuff[2].size() > 10:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
# else:
# smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
# smoothTipIndex = vals.depthBuff[2].back()
# # log the depth and index tip raw coordinate
# vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
# smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))
#Mouse Events
#doMouse.mouseActivities(pygame,rpt, tipIndex,tipThumb,kIndex,kThumb,m,k)
doMouse.clientMouseActivities(pygame,rpt,m,k)
#Gestures
# print doDepth.checkAllAboveBox()
# if doDepth.checkAllAboveBox():
# doGestures.gestures(averageX,averageY,k,m)
if vals.mouse_flg==1:
tipParam = 7
knuParam = 3
if vals.knuckleFlag:
fingerX = (rpt[tipIndex][0] * tipParam + rpt[kIndex][0] * knuParam) / (tipParam + knuParam)
fingerY = (rpt[tipIndex][1] * tipParam + rpt[kIndex][1] * knuParam) / (tipParam + knuParam)
else:
fingerX, fingerY = rpt[tipIndex][0], rpt[tipIndex][1]
# Relative:
if vals.relativeFlag:
smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
keyboardTest = myIsOnKeyboard(fingerX, fingerY, smoothTipIndex)
keyboardTest = isOnKeyboard(fingerX, fingerY, smoothTipIndex)
if vals.onKeyboardFlag and not keyboardTest:
# Hands up
vals.fingerBuff[0].erase()
vals.fingerBuff[1].erase()
# else:
# # Hands down
# pass
vals.onKeyboardFlag = keyboardTest
if vals.onKeyboardFlag:
vals.fingerBuff[0].put(fingerX)
vals.fingerBuff[1].put(fingerY)
# Move the mouse
if vals.fingerBuff[0].size() >= 2:
mouseX, mouseY = finger2MouseRelative(vals.fingerBuff[0].data, \
vals.fingerBuff[1].data, vals.traceX, vals.traceY)
vals.buff[0].put(mouseX)
vals.buff[1].put(mouseY)
smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))
# print fingerX, fingerY, mouseX, mouseY
vals.traceX, vals.traceY = smoothX, smoothY
if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
m.move(vals.traceX, vals.traceY)
# Absolute:
elif not vals.relativeFlag:
if not vals.zoom_flg:
vals.fX=rpt[tipIndex][0]
vals.fY=rpt[tipIndex][1]
if ((vals.inputX2-vals.inputX1)==0) or ((vals.inputY2-vals.inputY1)==0):
# Use default option
mouseX, mouseY = finger2Mouse(fingerX, fingerY, False)
# mouseX=(rpt[tipIndex][0]-600)*vals.width/vals.windowX
# mouseY=(rpt[tipIndex][1]-150)*vals.height/vals.windowY
else:
# Use the user prefered window size
mouseX, mouseY = finger2Mouse(fingerX, fingerY, True)
if not vals.zoom_flg:
vals.fingerX=fingerX
vals.fingerY=fingerY
vals.mouseX=mouseX
vals.mouseY=mouseY
"""Currently we have the setting such that if there is a single LED that is out of range then
the mouse wont move. The problem with this is that the range of the mouse gets limited, and
some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice
versa. So this is a give or take until we have a better filtering method."""
if (vals.inrange and doDepth.checkIndexInBox()) or vals.mouseState == vals.MOUSE_DRAG:
vals.buff[0].put(mouseX)
vals.buff[1].put(mouseY)
smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))
if vals.featureFlag:
vals.constBuff[0].put(mouseX)
vals.constBuff[1].put(mouseY)
sX = np.mean(fun.smooth(vals.constBuff[0].data, window_len = vals.constBuff[0].size()))
sY = np.mean(fun.smooth(vals.constBuff[1].data, window_len = vals.constBuff[1].size()))
# The speed of the cursor
speed = np.sqrt( (sX - vals.traceX)**2 + (sY - vals.traceY)**2 )
if speed < 0.0001:
speed = 0.0001
vals.speedBuff.put(speed)
vals.smoothSpeed = np.mean(fun.smooth(vals.speedBuff.data, window_len = vals.speedBuff.size()))
# Several method to get buffer size from speed:
# paramA, paramB = 8.5, 20
# 1) size = A + B / speed
# newSize = max(int(paramA + paramB / vals.smoothSpeed), vals.minBuffSize)
# 2) size = A + B / sqrt(speed)
# newSize = max(int(paramA + paramB / np.sqrt(vals.smoothSpeed)), vals.minBuffSize)
# newSize = min(newSize, vals.maxBuffSize)
# P1(minSpeed, maxBuff), P2(maxSpeed, minBuff)
maxSpeed = 25
minSpeed = 0.1
maxBuff = 35
minBuff = 10
# 3) size = A + B * speed
paramB = float(minBuff - maxBuff) / (maxSpeed - minSpeed)
paramA = maxBuff - paramB * minSpeed
newSize = paramA + paramB * vals.smoothSpeed
newSize = max(min(int(newSize), maxBuff), minBuff)
vals.buff[0].setBuffSize(newSize)
vals.buff[1].setBuffSize(newSize)
if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
# Record the last trace point
vals.traceX, vals.traceY = smoothX, smoothY
# if vals.featureFlag:# and vals.mouseState == vals.MOUSE_READY:
# # param = 20.0 / speed2
# # vals.traceX = int((vals.traceX * param + smoothX) / (1 + param))
# # vals.traceY = int((vals.traceY * param + smoothY) / (1 + param))
# vals.traceX = np.mean(fun.smooth(vals.buff[0].data[-vals.smoothSize:], window_len = vals.smoothSize))
# vals.traceY = np.mean(fun.smooth(vals.buff[1].data[-vals.smoothSize:], window_len = vals.smoothSize))
# # vals.traceX = (vals.traceX * 4 + smoothX) / 5
# # vals.traceY = (vals.traceY * 4 + smoothY) / 5
m.move(vals.traceX, vals.traceY)
# m.move(vals.buff[0].data[-1],vals.buff[1].data[-1])
# m.move(smoothX, smoothY)
# m.move(mouseX, mouseY)
if vals.wiimoteNum >= vals.wiimoteMaxNum \
and not (vals.calibLoadFlag or vals.calibration or vals.rec_flg):
doDraw.drawDefault(screen, defaultFont)
eventsObject=pygame.event.get()
doEvents.eventHandling(eventsObject)
msElapsed=clock.tick(pygameRate)
pygame.display.update()
if kill==True or vals.quit_FLG:
pygame.quit()
sys.exit()
break
for event in eventsObject:
if (event.type == QUIT) or (event.type == KEYDOWN and event.key == K_ESCAPE):
kill=True
pygame.quit()
sys.exit()
break
#Finding out the location of the LEDs, tipThumb, kThumb....
newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
averageX,averageY=findingPoints.centerFind(rpt) #the center point
#Find out the location of the 2nd Wiimote LEDs
newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
#Check whether LED is in range
newRpt=copy.deepcopy(rpt)
vals.rptList.append(newRpt)
vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
#Depth
doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
#GUI
<<<<<<< HEAD
#doDraw.drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,myfont,calibFont,depthFont)
doDraw.drawAllMiniRecording(miniScreen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,myfont,calibFont,depthFont)
#Mouse Events
doMouse.mouseActivities(rpt, tipIndex,tipThumb,kIndex,kThumb,m,k)
=======
doDraw.drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY)
#Mouse Events
doMouse.mouseActivities(rpt, tipIndex,tipThumb,kIndex,kThumb)
>>>>>>> 098035ea117798cf3908cf721c502fcebf4ee6a2
#Gestures
doGestures.gestures(averageX,averageY)
