Пример #1
0
def getMetric(target, output, metric="CCC"):
	if metric == "CCC":
		result = CCC(target, output)
	if metric == "MSE":
		result = MSE(target, output)
	if metric == "RMSE":
		result = RMSE(target, output)
	if metric == "Accuracy":
		result = Accuracy(target, output)
	if metric == "UAR":
		from sklearn.metrics import recall_score
		result = recall_score(target, output, average='macro')
	if metric == "AUC":
		from sklearn.metrics import roc_auc_score
		result = roc_auc_score(target, output)
	if "AUC-" in metric:
		from sklearn.metrics import roc_auc_score
		result = roc_auc_score(target, smooth(output, win=int(metric.split('-')[1])))
	if "SmoothAcc-" in metric:
		result = Accuracy(target, smooth(output, win=int(metric.split('-')[1])))
	return result
Пример #2
0
        if mouse_flg==1:
            X=rpt[tipIndex][0]
            mouseX=(X-600)*width/400                    
            Y=rpt[tipIndex][1]
            mouseY=(Y-150)*height/290

            """Currently we have the setting such that if there is a single LED that is out of range then
            the mouse wont move. The problem with this is that the range of the mouse gets limited, and 
            some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
            then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice 
            versa. So this is a give or take until we have a better filtering method."""

            if inrange:
                buff[0].put(mouseX)
                buff[1].put(mouseY)
                smoothX=np.mean(fun.smooth(buff[0].data, window_len=len(buff[0].data)))
                smoothY=np.mean(fun.smooth(buff[1].data, window_len=len(buff[1].data)))
                
                m.move(buff[0].data[-1],buff[1].data[-1])
#                 m.move(smoothX,smoothY)                    
################################################################################################################


    for event in pygame.event.get():
        if event.type==KEYDOWN:
            if event.key==pygame.K_r: #start recording
                rec_flg=1
                calibration=False
            elif event.key==pygame.K_c: #start calibration
                calibration=1
            elif event.key==pygame.K_s: #pauses the recording
Пример #3
0
    def run(self):
        global FLG, coords
        FLG = 1
        m = PyMouse()
        k = PyKeyboard()
        
        #Intialization of Pygame
        pygameRate=100
        os.environ['SDL_VIDEO_iWINDOW_POS'] = "%d,%d" % (0,0)   

        pygame.mixer.pre_init(44100, -16, 2, 2048) # setup mixer to avoid sound lag
        pygame.init()
        vals.switchSound = pygame.mixer.Sound('switch.wav')
        vals.clickSound = pygame.mixer.Sound('click.wav')

        clock=pygame.time.Clock()
        myfont=pygame.font.SysFont("monospace",15)
        calibFont=pygame.font.SysFont("monospace",20)
        depthFont=pygame.font.SysFont("monospace",10)
        defaultFont=pygame.font.SysFont("monospace",15)

        pygame.mixer.init()
        
        infoObject = pygame.display.Info()
        vals.width = infoObject.current_w
        vals.height = infoObject.current_h
        # print width, height
        screen=pygame.display.set_mode((int(vals.width*0.5), vals.height))
        global kill
        
        
        #Main loop
        while kill==False:
            
            screen.fill((0,0,0))
            rpt=[[int(i2) for i2 in i]for i in coords[0]]
            rpt2=[[int(i2) for i2 in i]for i in coords[1]]
            
            if vals.switchRPT:
                tmp = rpt
                rpt = rpt2
                rpt2 = tmp

            if not vals.rec_flg and (vals.calibration or vals.calibLoadFlag): #do calibration or load from file
            #Receiving data from the threads
                newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
                tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
                averageX,averageY=findingPoints.centerFind(rpt) #the center point
            #Find out the location of the 2nd Wiimote LEDs
                newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
                tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
            #GUI section
                #doDraw.drawAllMiniCalibration(miniScreen, rpt, tipIndex, tipThumb,kThumb,kIndex,averageX,averageY,myfont,calibFont,depthFont)
                doDraw.drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)
            

            if vals.inputCalibration:
            #Receiving data from the threads
                newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
                tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
                averageX,averageY=findingPoints.centerFind(rpt) #the center point
            #Find out the location of the 2nd Wiimote LEDs
                newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
                tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
            #GUI section
                doDraw.drawInputCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2, tipIndex2, tipThumb2,kThumb2,kIndex2, averageX,averageY,myfont,calibFont,depthFont)

                #first "click" get where the rpt index x,y, store it. Second click completes the line, turn into square
                distClick=fun.distanceVec(\
                [rpt[kIndex][0]],\
                [rpt[kIndex][1]],\
                [rpt[tipThumb][0]],\
                [rpt[tipThumb][1]])

                #first click
                if (distClick[0]<vals.clickValue and vals.inputCounter==0):
                    vals.inputX1=rpt[tipIndex][0]
                    vals.inputY1=rpt[tipIndex][1]
                    vals.inputCounter+=1
                    vals.inputClickStopper=1
                    print "awef"
                    print vals.inputX1,vals.inputY1
                if (distClick[0]>=vals.clickValue and vals.inputClickStopper):
                    vals.inputClickStopper=0
                
                #second click
                if (distClick[0]<vals.clickValue and vals.inputCounter==1 and not vals.inputClickStopper):
                    vals.inputX2=rpt[tipIndex][0]
                    vals.inputY2=rpt[tipIndex][1]
                    print "hey"
                    print vals.inputX2,vals.inputY2
                    vals.inputCounter+=1
                    vals.inputSet=1

                if vals.inputSet:
                    #sets the inrange constants
                    if vals.inputX1<=vals.inputX2:
                        vals.leftBound=vals.inputX1
                        vals.rightBound=vals.inputX2
                    else:
                        vals.leftBound=vals.inputX2
                        vals.rightBound=vals.inputX1                        

                    if vals.inputY1<=vals.inputY2:
                        vals.upperBound=vals.inputY1
                        vals.lowerBound=vals.inputY2
                    else:
                        vals.upperBound=vals.inputY2
                        vals.lowerBound=vals.inputY1       

                    vals.windowX = vals.rightBound - vals.leftBound
                    vals.windowY = vals.lowerBound - vals.upperBound                 
                    


            if vals.rec_flg==1: #Recording 
            #Finding out the location of the LEDs, tipThumb, kThumb....
                newList=findingPoints.findDegrees(rpt) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex, tipIndexAngle, kIndex,kIndexAngle=findingPoints.indexData(newList)
                tipThumb,tipThumbAngle,kThumb,kThumbAngle=findingPoints.thumbData(newList)
                averageX,averageY=findingPoints.centerFind(rpt) #the center point
            #Find out the location of the 2nd Wiimote LEDs
                newList2=findingPoints.findDegrees(rpt2) #returns in from [(theta1,i1),(theta2,i2)....)]
                tipIndex2, tipIndexAngle2, kIndex2,kIndexAngle2=findingPoints.indexData(newList2)
                tipThumb2,tipThumbAngle2,kThumb2,kThumbAngle2=findingPoints.thumbData(newList2)
                averageX2,averageY2=findingPoints.centerFind(rpt2) #the center point

            #Check whether LED is in range
                newRpt=copy.deepcopy(rpt)
                vals.rptList.append(newRpt)
                vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
            
                if (vals.inputX1<=rpt[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt[tipIndex][1]<=vals.inputY2) or \
                    (vals.inputX1<=rpt2[tipIndex][0]<=vals.inputX2 and vals.inputY1<=rpt2[tipIndex][1]<=vals.inputY2):
                    vals.inrange=1

            #Depth
                doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
            #GUI
                doDraw.drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,averageX2,averageY2,myfont,calibFont,depthFont)
                #doDraw.drawAllMiniRecording(miniScreen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,myfont,calibFont,depthFont)

                # Only log the depth data in record mode when testing or debugging
                # if vals.relativeFlag:
                #     if vals.depthBuff[2].size() > 10:
                #         smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
                #     else:
                #         smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
                #     smoothTipIndex = vals.depthBuff[2].back()
                #     # log the depth and index tip raw coordinate
                #     vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
                #         smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))

            #Mouse Events
                #doMouse.mouseActivities(pygame,rpt, tipIndex,tipThumb,kIndex,kThumb,m,k)
                doMouse.clientMouseActivities(pygame,rpt,m,k)

            #Gestures
                # print doDepth.checkAllAboveBox()
                # if doDepth.checkAllAboveBox():
                #     doGestures.gestures(averageX,averageY,k,m)

                if vals.mouse_flg==1:
                    tipParam = 7
                    knuParam = 3
                    if vals.knuckleFlag:
                        fingerX = (rpt[tipIndex][0] * tipParam + rpt[kIndex][0] * knuParam) / (tipParam + knuParam)
                        fingerY = (rpt[tipIndex][1] * tipParam + rpt[kIndex][1] * knuParam) / (tipParam + knuParam)
                    else:
                        fingerX, fingerY = rpt[tipIndex][0], rpt[tipIndex][1]

                    # Relative:
                    if vals.relativeFlag:
                        smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
                        keyboardTest = myIsOnKeyboard(fingerX, fingerY, smoothTipIndex)
                        keyboardTest = isOnKeyboard(fingerX, fingerY, smoothTipIndex)

                        if vals.onKeyboardFlag and not keyboardTest:
                            # Hands up
                            vals.fingerBuff[0].erase()
                            vals.fingerBuff[1].erase()
                        # else:
                        #     # Hands down
                        #     pass

                        vals.onKeyboardFlag = keyboardTest

                        if vals.onKeyboardFlag:
                            vals.fingerBuff[0].put(fingerX)
                            vals.fingerBuff[1].put(fingerY)

                            # Move the mouse
                            if vals.fingerBuff[0].size() >= 2:
                                mouseX, mouseY = finger2MouseRelative(vals.fingerBuff[0].data, \
                                    vals.fingerBuff[1].data, vals.traceX, vals.traceY)

                                vals.buff[0].put(mouseX)
                                vals.buff[1].put(mouseY)
                             
                                smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
                                smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))

                                # print fingerX, fingerY, mouseX, mouseY
                                vals.traceX, vals.traceY = smoothX, smoothY

                            if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
                                m.move(vals.traceX, vals.traceY)                     

                    # Absolute:
                    elif not vals.relativeFlag:
                        if not vals.zoom_flg:
                            vals.fX=rpt[tipIndex][0]
                            vals.fY=rpt[tipIndex][1]

                        if ((vals.inputX2-vals.inputX1)==0) or ((vals.inputY2-vals.inputY1)==0):
                            # Use default option
                            mouseX, mouseY = finger2Mouse(fingerX, fingerY, False)
                            # mouseX=(rpt[tipIndex][0]-600)*vals.width/vals.windowX                    
                            # mouseY=(rpt[tipIndex][1]-150)*vals.height/vals.windowY
                        else:
                            # Use the user prefered window size
                            mouseX, mouseY = finger2Mouse(fingerX, fingerY, True)
            

                        if not vals.zoom_flg:
                            vals.fingerX=fingerX
                            vals.fingerY=fingerY
                            vals.mouseX=mouseX
                            vals.mouseY=mouseY




                        """Currently we have the setting such that if there is a single LED that is out of range then
                        the mouse wont move. The problem with this is that the range of the mouse gets limited, and 
                        some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
                        then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice 
                        versa. So this is a give or take until we have a better filtering method."""
            
                        if (vals.inrange and doDepth.checkIndexInBox()) or vals.mouseState == vals.MOUSE_DRAG:
                            vals.buff[0].put(mouseX)
                            vals.buff[1].put(mouseY)
                         
                            smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len = vals.buff[0].size()))
                            smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len = vals.buff[1].size()))

                            if vals.featureFlag:
                                vals.constBuff[0].put(mouseX)
                                vals.constBuff[1].put(mouseY)

                                sX = np.mean(fun.smooth(vals.constBuff[0].data, window_len = vals.constBuff[0].size()))
                                sY = np.mean(fun.smooth(vals.constBuff[1].data, window_len = vals.constBuff[1].size()))


                                # The speed of the cursor
                                speed = np.sqrt( (sX - vals.traceX)**2 + (sY - vals.traceY)**2 )
                                if speed < 0.0001:
                                    speed = 0.0001
                                vals.speedBuff.put(speed)
                                vals.smoothSpeed = np.mean(fun.smooth(vals.speedBuff.data, window_len = vals.speedBuff.size()))

                                # Several method to get buffer size from speed:
                                
                                # paramA, paramB = 8.5, 20

                                # 1) size = A + B / speed
                                # newSize = max(int(paramA + paramB / vals.smoothSpeed), vals.minBuffSize)

                                # 2) size = A + B / sqrt(speed)
                                # newSize = max(int(paramA + paramB / np.sqrt(vals.smoothSpeed)), vals.minBuffSize)
                                
                                # newSize = min(newSize, vals.maxBuffSize)

                                # P1(minSpeed, maxBuff), P2(maxSpeed, minBuff)
                                maxSpeed = 25
                                minSpeed = 0.1
                                maxBuff = 35
                                minBuff = 10

                                # 3) size = A + B * speed
                                paramB = float(minBuff - maxBuff) / (maxSpeed - minSpeed)
                                paramA = maxBuff - paramB * minSpeed

                                newSize = paramA + paramB * vals.smoothSpeed
                                newSize = max(min(int(newSize), maxBuff), minBuff)
                                
                                vals.buff[0].setBuffSize(newSize)
                                vals.buff[1].setBuffSize(newSize)

                            if not vals.testTypeFlag or (vals.testTypeFlag and vals.testPointFlag):
                                # Record the last trace point
                                vals.traceX, vals.traceY = smoothX, smoothY
                                # if vals.featureFlag:# and vals.mouseState == vals.MOUSE_READY:
                                #     # param = 20.0 / speed2
                                #     # vals.traceX = int((vals.traceX * param + smoothX) / (1 + param))
                                #     # vals.traceY = int((vals.traceY * param + smoothY) / (1 + param))
                                #     vals.traceX = np.mean(fun.smooth(vals.buff[0].data[-vals.smoothSize:], window_len = vals.smoothSize))
                                #     vals.traceY = np.mean(fun.smooth(vals.buff[1].data[-vals.smoothSize:], window_len = vals.smoothSize))
                                #     # vals.traceX = (vals.traceX * 4 + smoothX) / 5
                                #     # vals.traceY = (vals.traceY * 4 + smoothY) / 5

                                m.move(vals.traceX, vals.traceY)
                                # m.move(vals.buff[0].data[-1],vals.buff[1].data[-1])
                                # m.move(smoothX, smoothY)
                                # m.move(mouseX, mouseY)
            
            if vals.wiimoteNum >= vals.wiimoteMaxNum \
            and not (vals.calibLoadFlag or vals.calibration or vals.rec_flg):
                doDraw.drawDefault(screen, defaultFont)

            eventsObject=pygame.event.get()
            doEvents.eventHandling(eventsObject)
            
            msElapsed=clock.tick(pygameRate)
            pygame.display.update() 
            
            
             
            if kill==True or vals.quit_FLG:
                pygame.quit()
                sys.exit()
                break
            
            for event in eventsObject: 
                if (event.type == QUIT) or (event.type == KEYDOWN and event.key == K_ESCAPE):
                    kill=True
                    pygame.quit()
                    sys.exit()
                    break
Пример #4
0
def drawAllCalibration(screen, rpt, tipIndex, tipThumb,kThumb,kIndex,rpt2,tipIndex2, tipThumb2,kThumb2,kIndex2,averageX,averageY,myfont,calibFont,depthFont):
    mouseModeDistance=fun.distanceVec(\
        [rpt[tipIndex][0]],\
        [rpt[tipIndex][1]],\
        [rpt[tipThumb][0]],\
        [rpt[tipThumb][1]])

    clickingDistance=fun.distanceVec(\
        [rpt[kIndex][0]],\
        [rpt[kIndex][1]],\
        [rpt[tipThumb][0]],\
        [rpt[tipThumb][1]])


    screen.fill(vals.black)
    #Drawing the Circles
    pygame.draw.circle(screen, vals.yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
    pygame.draw.circle(screen, vals.red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
    pygame.draw.circle(screen, vals.green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
    pygame.draw.circle(screen, vals.blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
    pygame.draw.circle(screen, vals.white, (averageX/3,averageY/3),10)

    #Drawing the instructions
    pygame.draw.rect(screen, vals.gray, (0,5,600,60))
    if vals.calibState == vals.START_CALIB:
        Calib1=calibFont.render("Press H to calibrate the switching gesture",1,vals.black)
        screen.blit(Calib1,(0,15))

    elif vals.calibState == vals.MOUSE_MODE_CALIB:
        Calib1=calibFont.render("Tap tip of thumb and tip of index",1,vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press H to complete",1,vals.black)
        screen.blit(Calib2,(0,35))
        pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),5 )
        
        vals.mouseModeCalibList.append(mouseModeDistance[0])     
        doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
        vals.boxBoundCalibList.append(doDepth.meanDepth())       

        # Use 3d-box instead just using z-axis information
        if vals.depthBuff[2].size() == 10:
        #     smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
        # else:
            smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
            # smoothTipIndex = vals.depthBuff[2].back()
            # log the depth and index tip raw coordinate
            vals.switchBoxData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
                smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1])) 

    elif vals.calibState == vals.READY_CLICK_CALIB:
        Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press H to calibrate the clicking gesture",1,vals.black)
        screen.blit(Calib2,(0,35))

    elif vals.calibState == vals.CLICK_CALIB:
        Calib1=calibFont.render("Tap tip of thumb and knuckle of index for {} times".format(\
            str(vals.clickNum)), 1, vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press H to complete",1,vals.black)
        screen.blit(Calib2,(0,35))
        pygame.draw.line(screen,vals.white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[kIndex][0]/3,rpt[kIndex][1]/3),5 )
        
        vals.clickingCalibList[0].append(time.time() - vals.clickCalibSTime)
        vals.clickingCalibList[1].append(clickingDistance[0]) 

    elif vals.calibState == vals.READY_DEPTH_CALIB:
        Calib1=calibFont.render("Put your hand on the keyboard",1,vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press H to calibrate keyboard plane",1,vals.black)
        screen.blit(Calib2,(0,35))

    elif vals.calibState == vals.DEPTH_CALIB:
        Calib1=calibFont.render("Draw 5 circles on the keyboard",1,vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press H to complete",1,vals.black)
        screen.blit(Calib2,(0,35))

        # Show the depth of the four LED : 440, 460, 480, 500
        ledDepthKey = ['tipThumb: ', 'knuThumb: ', 'tipIndex: ', 'knuIndex: ']
        for i in range(len(ledDepthKey)):
            ledDepth = calibFont.render(ledDepthKey[i] + str(vals.depthBuff[i].back()), 1, vals.white)
            screen.blit(ledDepth, (0, 80 + i * 20))

        doDepth.findingDepth(rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2)
        if vals.depthBuff[2].size() == 10:
        #     smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data[-10:], window_len = 10))
        # else:
            smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
            # smoothTipIndex = vals.depthBuff[2].back()
            # log the depth and index tip raw coordinate
            vals.planeDepthData.append('{}, {}, {}, {}, {}, {}'.format(vals.depthBuff[0].back(), vals.depthBuff[1].back(), \
                smoothTipIndex, vals.depthBuff[3].back(), rpt[tipIndex][0], rpt[tipIndex][1]))

                      
    elif vals.calibState == vals.END_CALIB:
        calibrationDone=1
        Calib1=calibFont.render("Calibration Completed",1,vals.black)
        screen.blit(Calib1,(0,15))
        Calib2=calibFont.render("Press r to start recording",1,vals.black)
        screen.blit(Calib2,(0,35))
Пример #5
0
def drawAllRecording(screen, rpt, rpt2, tipThumb,tipThumb2, kThumb,kThumb2, tipIndex,tipIndex2,kIndex,kIndex2,averageX,averageY,averageX2,averageY2,myfont, calibFont,depthFont):
#     screen.fill(vals.black)
    if not vals.testTypeFlag:
    # Normal Recording Mode
        mouseLabel=myfont.render("Mouse:"+" "+str(vals.mouseModeValue) ,1,(255,255,255))
        screen.blit(mouseLabel,(0,80))
        clickLabel=myfont.render("Click:"+" "+str(vals.clickValue) ,1,(255,255,255))
        screen.blit(clickLabel,(0,95))        

        Calib1=calibFont.render("tipThumb:"+str(int(vals.depthBuff[0].mean())),1,vals.white)
        screen.blit(Calib1,(0,115))

        Calib2=calibFont.render("kThumb:"+str(int(vals.depthBuff[1].mean())),1,vals.white)
        screen.blit(Calib2,(0,135))

        Calib3=calibFont.render("tipIndex:"+str(int(vals.depthBuff[2].mean())),1,vals.white)
        screen.blit(Calib3,(0,155))
        
        Calib4=calibFont.render("kIndex:"+str(int(vals.depthBuff[3].mean())),1,vals.white)
        screen.blit(Calib4,(0,175))

        #tipDistance=calibFont.render("Switch-Distance:"+str(int(vals.tipDistance)),1,vals.white)
        #screen.blit(tipDistance,(0,205))

        #clickDistance=calibFont.render("Click-Distance:"+str(int(vals.clickDistance)),1,vals.white)
        #screen.blit(clickDistance,(0,235))
        x,y = rpt[tipIndex][0], rpt[tipIndex][1]
        z = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))

        distance = getPlaneDistance(vals.planeParam, x, y, z)
        keyboardTop = vals.planeParam[-1]

        tipDistance=calibFont.render("Distance:"+str((distance)),1,vals.white)
        screen.blit(tipDistance,(0,205))

        clickDistance=calibFont.render("Keyboard Top:"+str((keyboardTop)),1,vals.white)
        screen.blit(clickDistance,(0,235))




        distance3D=calibFont.render("3D-Distance:"+str(int(vals.dist3D)),1,vals.white)
        screen.blit(distance3D,(0,255))

        box3D=calibFont.render(str(int(vals.boxLimit)),1,vals.white)
        screen.blit(box3D,(0,285))

#Circles to check on mode switching
        # if doDepth.checkAllInBox(): # Used to log data
        if vals.inSwitchBox:
            pygame.draw.circle(screen, vals.green, (10,325),10)
        else:
            pygame.draw.circle(screen, vals.red, (10,325),10)

        inBox=calibFont.render("in 3dBox",1,vals.white)
        screen.blit(inBox,(20,325))

        vals.inrange, vals.LED1,vals.LED2,vals.LED3,vals.LED4=checkingInRange.rangeChecker(vals.rptList, vals.LED1, vals.LED2,vals.LED3,vals.LED4)
        if (vals.inrange==1):
            pygame.draw.circle(screen, vals.green, (10,345),10)
        else:
            pygame.draw.circle(screen, vals.red, (10,345),10)
        
        inBox=calibFont.render("inrange",1,vals.white)
        screen.blit(inBox,(20,345))

        if (not vals.mouseSwitched_flg): # Used to log data
            pygame.draw.circle(screen, vals.green, (10,365),10)
        else:
            pygame.draw.circle(screen, vals.red, (10,365),10)
        
        inBox=calibFont.render("open tips",1,vals.white)
        screen.blit(inBox,(20,365))

        # Show window size which is used to control the sensitivity
        windowSize = calibFont.render('window size X:{} Y:{}'.format(str(vals.windowX), str(vals.windowY)), 1, vals.white)
        screen.blit(windowSize, (0, 400))

        # Show the buffer size which is modified by the speed in the new feature(shift+f to turn on/off)
        if vals.featureFlag:
            buffSize = calibFont.render('current buffer size: {}'.format(str(vals.buff[0].size())), 1, vals.white)
        else:
            buffSize = calibFont.render('default buffer size: {}'.format(str(vals.defaultBuffSize)), 1, vals.white)
        screen.blit(buffSize, (0, 420))

        # Show the depth of the four LED : 440, 460, 480, 500
        ledDepthKey = ['tipThumb: ', 'knuThumb: ', 'tipIndex: ', 'knuIndex: ']
        for i in range(len(ledDepthKey)):
            ledDepth = calibFont.render(ledDepthKey[i] + str(vals.depthBuff[i].back()), 1, vals.white)
            screen.blit(ledDepth, (0, 440 + i * 20))

        if vals.relativeFlag:
            controlMode = calibFont.render('Relative Mode', 1, vals.white)
            screen.blit(controlMode, (0, 520))

            onKeyboard = calibFont.render('Keyboard Test: ' + str(vals.onKeyboardFlag), 1, vals.white)
            screen.blit(onKeyboard, (0, 540))
        else:
            controlMode = calibFont.render('Absolute Mode', 1, vals.white)
            screen.blit(controlMode, (0, 520))

        rawXY = calibFont.render('Raw X:{}, Y:{}'.format(rpt[tipIndex][0], rpt[tipIndex][1]), 1, vals.white)
        screen.blit(rawXY, (0, 560))

        traceXY = calibFont.render('Mouse X:{}, Y:{}'.format(int(vals.traceX), int(vals.traceY)), 1, vals.white)
        screen.blit(traceXY, (0, 580))

        smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
        distance = getPlaneDistance(vals.planeParam, rpt[tipIndex][0], rpt[tipIndex][1], smoothTipIndex)
        tipIndexDepth = calibFont.render('Tip Index Depth: {}'.format(distance), 1, vals.white)
        screen.blit(tipIndexDepth, (0, 600))

    #main circles
        pygame.draw.circle(screen, vals.red, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
        pygame.draw.circle(screen, vals.blue, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
        pygame.draw.circle(screen, vals.green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
        pygame.draw.circle(screen, vals.white, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)

        pygame.draw.circle(screen, vals.gray, (averageX/3,averageY/3),13)
        pygame.draw.circle(screen, vals.gray, (averageX2/3,averageY2/3),13)

        pygame.draw.circle(screen, vals.red, (rpt2[tipIndex2][0]/3,rpt2[tipIndex2][1]/3),10)
        pygame.draw.circle(screen, vals.blue, (rpt2[kIndex2][0]/3,rpt2[kIndex2][1]/3),10)
        pygame.draw.circle(screen, vals.green, (rpt2[tipThumb2][0]/3,rpt2[tipThumb2][1]/3),10)
        pygame.draw.circle(screen, vals.white, (rpt2[kThumb2][0]/3,rpt2[kThumb2][1]/3),10)

    #GUI for depth
        # pygame.draw.rect(screen, vals.gray, (500,0,1500,1500))
        depthGUILeft = int(vals.width * 0.35)
        depthGUITop = int(vals.height * 0.05)
        depthGUIWidth = int(vals.width * 0.45) - depthGUILeft
        depthGUIHeight = int(vals.height * 0.95) - depthGUITop
        pygame.draw.rect(screen, vals.gray, (depthGUILeft, depthGUITop, depthGUIWidth, depthGUIHeight))
        
        # 0left--1startPos--2green--3white--4red--5blue--6endPos--7label--8right
        objXPos = np.linspace(depthGUILeft, depthGUILeft + depthGUIWidth, 9)
        objXPos = [int(x) for x in objXPos]
        #Creating the lines
        for i in xrange(11):
            offsetY = 75
            startPos = (objXPos[1], offsetY + i*30)      # was 550
            endPos = (objXPos[6], offsetY + i*30)      # was 650 
            pygame.draw.line(screen, vals.black, startPos, endPos)
            depthLabel = depthFont.render( str(5*i), 1, vals.black)
            screen.blit(depthLabel, (objXPos[7], offsetY + i*30))
        #Depth circles. Was 560, 580, 600, 620
        pygame.draw.circle(screen, vals.green, (objXPos[2], int(75 + vals.depthBuff[0].mean()*6)), 10) #tipThumb
        pygame.draw.circle(screen, vals.white, (objXPos[3], int(75 + vals.depthBuff[1].mean()*6)), 10) #kThumb
        pygame.draw.circle(screen, vals.red, (objXPos[4], int(75 + vals.depthBuff[2].mean()*6)), 10) #tipindex
        pygame.draw.circle(screen, vals.blue, (objXPos[5], int(75 + vals.depthBuff[3].mean()*6)), 10)#kIndex

    #The gesture bounds
        pygame.draw.line(screen,vals.red, (vals.gestureLeftThreshHold/3,0),(vals.gestureLeftThreshHold/3,800))
        pygame.draw.line(screen,vals.blue, (0,vals.gestureDownThreshHold/3),(10000,vals.gestureDownThreshHold/3))
        pygame.draw.line(screen,vals.yellow, (0,vals.gestureUpThreshHold/3),(10000,vals.gestureUpThreshHold/3))
    #Mouses mode drawing
        if vals.mouse_flg:
            MouseKeyboard=myfont.render( "Mouse mode",1,(255,255,255))
        else:
            MouseKeyboard=myfont.render( "Keyboard mode",1,(255,255,255))
        screen.blit(MouseKeyboard,(0,50))

        #input rectangle
        pygame.draw.line(screen,vals.white, (vals.inputX1/3,vals.inputY1/3), (vals.inputX2/3,vals.inputY1/3))
        pygame.draw.line(screen,vals.white, (vals.inputX2/3,vals.inputY1/3), (vals.inputX2/3,vals.inputY2/3))
        pygame.draw.line(screen,vals.white, (vals.inputX2/3,vals.inputY2/3), (vals.inputX1/3,vals.inputY2/3))
        pygame.draw.line(screen,vals.white, (vals.inputX1/3,vals.inputY2/3), (vals.inputX1/3,vals.inputY1/3))
        
    else:
    # Testing Recording Mode: Show article text.
    # I canceled the text input area.

        testFont = 'textFiles/MonospaceTypewriter.ttf'
        # Text GUI on the left: show the text
        
        if vals.textContent == '':
        # Read from file        
            try:
                tcf = open(vals.textContentFile, 'r')
                vals.textContent = tcf.read()
                tcf.close()

                print vals.textContent
            except:
                print 'Error: Invalid type content file.'

        
        textGUILeft = int(vals.width * 0.05)
        textGUITop = int(vals.height * 0.05)
        textGUIWidth = int(vals.width * 0.45) - textGUILeft
        textGUIHeight = int(vals.height * 0.95) - textGUITop
        pygame.draw.rect(screen, vals.white, (textGUILeft, textGUITop, textGUIWidth, textGUIHeight))

        if vals.textGUI == None:
            textFontSize = 20
            textBorder = 10

            vals.textGUI = Reader(unicode(vals.textContent.expandtabs(4), 'utf8'), (textGUILeft + textBorder, textGUITop + textBorder), \
                textGUIWidth - 2 * textBorder, textFontSize, height = textGUIHeight - 2 * textBorder, \
                font = testFont, fgcolor = (0, 0, 0), hlcolor = (255,10,150,100), split = True)

        vals.textGUI.show()

        '''
Пример #6
0
def mouseActivities(pygame, rpt, tipIndex,tipThumb,kIndex,kThumb,m,k):
#3D Distance from the tipIndex to tipThumb
    dist3D=euclidean(rpt[tipIndex],rpt[tipThumb])
    vals.dist3D=dist3D   
    
#Distance for switching modes
    dista=fun.distanceVec(\
    [rpt[tipIndex][0]],\
    [rpt[tipIndex][1]],\
    [rpt[tipThumb][0]],\
    [rpt[tipThumb][1]])

    vals.tipDistance=dista[0]

#Distance for clicking - thumb tip to index knuckle
    distClick=fun.distanceVec(\
    [rpt[kIndex][0]],\
    [rpt[kIndex][1]],\
    [rpt[tipThumb][0]],\
    [rpt[tipThumb][1]])

    vals.clickDistance=distClick[0]


#Modifying vals.mouseModeValue with respect to distance between knuckles
   # currentKnuckleValue=fun.distanceVec(\
   # [rpt[kIndex][0]],\
   # [rpt[kIndex][1]],\
   # [rpt[kThumb][0]],\
   # [rpt[kThumb][1]])[0]

   # knuckleRatio=float(currentKnuckleValue/vals.knuckleValue)
   # if knuckleRatio>1:
   #     newMouseModeValue=int(knuckleRatio*vals.mouseModeValue)
   #     newClickValue=int(knuckleRatio*vals.clickValue)
   # else:
    newMouseModeValue=vals.mouseModeValue
    newClickValue=vals.clickValue

    smoothTipIndex = np.mean(fun.smooth(vals.depthBuff[2].data, window_len = vals.depthBuff[2].size()))
    checkSwitchBox(rpt[tipIndex][0], rpt[tipIndex][1], smoothTipIndex)
    # inBox = doDepth.checkAllInBox() # Used to log data
    if vals.inSwitchBox:
    #Switching Modes
        #When distance tips goes below mouseModevalue, start measuring time.
        if 10<=dista[0]<=newMouseModeValue and vals.inrange==1 and vals.mouseModeSwitchTime==0:     
            vals.mouseModeSwitchTime=time.time()  

        hold5ms = (vals.timeHold<=(time.time()-vals.mouseModeSwitchTime)*1000) 
        tipInRange= (10<=dista[0]<=newMouseModeValue)
        mouseCondition= hold5ms and tipInRange# and vals.inrange==1
        #if distance is below for a certain time and all other conditions are met, then switch
        if mouseCondition and vals.mouse_flg==0 and not vals.mouseSwitched_flg:
            print('Mouse mode activated')
            vals.traceX, vals.traceY = m.position()

            vals.mouse_flg=1
            vals.mouseModeSwitchTime=0
            vals.mouseSwitched_flg=1

            try:
                vals.switchSound.play()
            except:
                pass
            # file = 'switch.mp3'
            
            # try:
            #     vals.a.play()
            # except:
            #     vals.a.load(file)
            #     vals.a.play()


        if mouseCondition and vals.mouse_flg==1 and not vals.mouseSwitched_flg:
            print('Mouse mode deactivated')
            vals.mouse_flg=0
            vals.contDist=0
            vals.mouseSwitched_flg=1

            try:
                vals.switchSound.play()
            except:
                pass

            # file = 'switch.mp3'
            # try:
            #     vals.a.play()
            # except:
            #     vals.a.load(file)
            #     vals.a.play()

        #after switching, the fingers need to part in order to reset constants.
        if (vals.mouseSwitched_flg and dista[0]>newMouseModeValue):
            vals.mouseSwitched_flg=0
            vals.mouseModeSwitchTime=0





    '''
    #Adjusting MaxBuff with respect to thumbtip and index knuckle
    # It doesn't work. Currently this method uses a larger buffer when clicking,
    # in order to avoid mistakes when getting the clickX and clickY.
    # But this method will only work when we put the cursor still on the target.
    # If we move and press 'click', it will be worse, because of its larger buffer contains
    # more wrong information.

    # Method 1: Sin
    # if vals.mouse_flg and vals.debugFlag:
    #     currBuff = vals.minBuff
    #     if distClick[0] < 0.8 * newClickValue:
    #         currBuff = vals.maxBuff            
    #     elif distClick[0] > 1.2 * newClickValue:
    #         currBuff = vals.minBuff
    #     else:
    #         currBuff = (vals.maxBuff + vals.minBuff)/2 - (vals.maxBuff - vals.minBuff)/2 * \
    #         np.sin((distClick[0] - newClickValue) * 2 * np.pi / 0.8 / newClickValue)
    #     vals.buff[0].setCurrBuff(currBuff)
    #     vals.buff[1].setCurrBuff(currBuff)

    #    a=40*newClickValue
    #    vals.maxBuff=a/distClick[0]
    #    if vals.maxBuff<20:
    #        vals.maxBuff=20
    #    elif vals.maxBuff>40:
    #        vals.maxBuff=40
    '''
    
    # Clicking and Dragging
    if vals.mouseState == vals.MOUSE_NORMAL:
        # print 'NORMAL'
        #print distClick[0], vals.inrange, vals.mouse_flg
        if distClick[0] <= newClickValue and vals.inrange and vals.mouse_flg:
            # Get possible point of click or drag
            vals.clickX, vals.clickY = vals.traceX, vals.traceY
            # vals.clickX = np.mean(fun.smooth(vals.buff[0].data, window_len=len(vals.buff[0].data)))
            # vals.clickY = np.mean(fun.smooth(vals.buff[1].data, window_len=len(vals.buff[1].data)))
            vals.dragX, vals.dragY = vals.clickX, vals.clickY

            vals.stime = time.time()
            vals.mouseState = vals.MOUSE_READY
            #vals.mouseActBuff = [[], []]
            print 'READY'
            print 'distClick[0]: ' + str(distClick[0])

    elif vals.mouseState == vals.MOUSE_READY:
        # print 'READY'
        currTime = (time.time() - vals.stime) * float(1000)
        # Note: We don't need to add time threshold here to do READY -> CLICK.
        # Because the point is discrete, it may jump from the READY rectangle to NORMAL without
        # triggering CLICK or DRAG signal. That is, its distClick < clickValue and time < timeThre for
        # the last point, but distClick > clickValue and time > timeThre for the current point.
        # In this case, we assume that is a CLICK. Noted by Zhen Li, Aug 5th, 2014.
        if distClick[0] > newClickValue and vals.mouse_flg and vals.inrange:# and currTime <= vals.mouseActTimeThre:
            # Click
            vals.mouseState = vals.MOUSE_CLICK
            # if not vals.testTypeFlag:
            #     m.click(vals.clickX, vals.clickY)

            # Detect double click:
            clickTime = (time.time() - vals.lastClickTime) * float(1000)
            if clickTime > vals.doubleClickTimeThre or not vals.dragFlag:
                # Click
                if not vals.testTypeFlag:
                    m.click(vals.clickX, vals.clickY)
                    vals.lastClickX, vals.lastClickY = vals.clickX, vals.clickY
                    print('Click')

                    try:
                        vals.clickSound.play()
                    except:
                        pass

                    # file = 'click.mp3'    
                    # try:
                    #     vals.b.play()
                    # except:
                    #     vals.b.load(file)
                    #     vals.b.play()

            else:
                # Double Click
                if not vals.testTypeFlag:
                    m.click(vals.lastClickX, vals.lastClickY)
                    print('Double Click')

                    try:
                        vals.clickSound.play()
                    except:
                        pass

                    # file = 'click.mp3'    
                    # try:
                    #     vals.b.play()
                    # except:
                    #     vals.b.load(file)
                    #     vals.b.play()

            vals.lastClickTime = time.time()
            # print('Click')
            print 'distClick[0]: ' + str(distClick[0])

        elif distClick[0] <= newClickValue and vals.mouse_flg and vals.inrange and currTime > vals.mouseActTimeThre:
            # Drag if enabled
            if not vals.testTypeFlag:
                if vals.dragFlag:
                    m.press(vals.dragX, vals.dragY)
                else:
                    m.click(vals.clickX, vals.clickY)
                    try:
                        vals.clickSound.play()
                    except:
                        pass
            vals.mouseState = vals.MOUSE_DRAG
            print('Drag')
            print 'distClick[0]: ' + str(distClick[0])

    elif vals.mouseState == vals.MOUSE_CLICK:
        # print 'CLICK'
        # vals.mouseState = vals.MOUSE_NORMAL
        vals.mouseState = vals.MOUSE_WAIT

    elif vals.mouseState == vals.MOUSE_WAIT:
        if distClick[0] > 1.5 * newClickValue and vals.mouse_flg and vals.inrange:
            vals.mouseState = vals.MOUSE_NORMAL

    elif vals.mouseState == vals.MOUSE_DRAG:
        # print 'DRAG'
        if distClick[0] > newClickValue and vals.mouse_flg:
            # Release if enabled
            if vals.dragFlag and not vals.testTypeFlag:
                m.release(vals.buff[0].mean(),vals.buff[1].mean())
            vals.mouseState = vals.MOUSE_NORMAL
            print("Release")
            print 'distClick[0]: ' + str(distClick[0])

    if vals.testTypeFlag or vals.testPointFlag:
        ''' TODO: convert them to actual coordinate on the display.'''
        tIX, tIY = finger2Mouse(rpt[tipIndex][0], rpt[tipIndex][1])
        kIX, kIY = finger2Mouse(rpt[kIndex][0], rpt[kIndex][1])
        tTX, tTY = finger2Mouse(rpt[tipThumb][0], rpt[tipThumb][1])
        kTX, kTY = finger2Mouse(rpt[kThumb][0], rpt[kThumb][1])
        smoothX, smoothY = vals.traceX, vals.traceY
        # smoothX = np.mean(fun.smooth(vals.buff[0].data, window_len=len(vals.buff[0].data)))
        # smoothY = np.mean(fun.smooth(vals.buff[1].data, window_len=len(vals.buff[1].data)))

        # time, dista0, distClick0, inrange, inBox
        # tIX, tIY, kIX, kIY, tTX, tTY, kTX, kTY, smoothX, smoothY
        # mouse_flg, mouseState, clickX, clickY, speed, buffSize
        vals.testTypeData.append('{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}'.format(\
            str(time.time() - vals.testStartTime), str(dista[0]), str(distClick[0]), str(int(vals.inrange)), str(int(vals.inSwitchBox)), \
            str(tIX), str(tIY), str(kIX), str(kIY), str(tTX), str(tTY), str(kTX), str(kTY), str(smoothX), str(smoothY), \
            str(vals.mouse_flg), str(vals.mouseState), str(vals.clickX), str(vals.clickY), str(vals.smoothSpeed), str(vals.buff[0].size())
            ))
Пример #7
0
def getDistAndTime(X, Y, paramD, paramT):
    "Get mean and std of distance and time(ms) with calib data: meanDist, stdDist, meanTime, stdTime"
    "Then get answer = mean + std * param"
    # plot(X, Y, 'g.')
    windowLen = 16
    smoothY = smooth(Y, windowLen)
    smoothY = smoothY[(windowLen / 2 - 1) : -(windowLen / 2)]
    # plot(X, smoothY, 'c-')

    # Note: The author suggests: '(sample / period) / f' where '4 >= f >= 1.25'
    f = 6
    lookahead = len(X) / vals.clickNum / f

    _max, _min = peakdetect(smoothY, X, lookahead)#, 0.30)
    # print _max
    # print _min
    xm = [p[0] for p in _max]
    ym = [p[1] for p in _max]
    xn = [p[0] for p in _min]
    yn = [p[1] for p in _min]

    # plot(xm, ym, 'rx', markersize = 10)
    # plot(xn, yn, 'kx', markersize = 10)

    yTh = getYTh(ym, yn)
    # print 'yTh: ' + str(yTh)

    newXMax = []
    newYMax = []
    newXMin = []
    newYMin = []

    i, j = 0, 0
    
    lastPoint = 0
    ADD_MAX, \
    ADD_MIN = range(2)

    if xm[i] < xn[j]:
        #max,min,...
        newXMax.append(xm[i])
        newYMax.append(ym[i])
        i = i + 1
        lastPoint = ADD_MAX
    else:
        #min, max,... : add a max point
        newXMax.append(X[0])
        newYMax.append(smoothY[0])
        lastPoint = ADD_MAX

    while i < len(_max) and j < len(_min):
        if lastPoint == ADD_MAX:
            if xm[i] > xn[j]:
                # print 'max, [min], max: ' + str(ym[i-1] - yn[j])
                if (ym[i-1] - yn[j]) > yTh:
                    #...max, [min], max... : Normal
                    # print 'add min:{}, x={}'.format(str(j), xn[j])
                    newXMin.append(xn[j])
                    newYMin.append(yn[j])
                    j = j + 1
                    lastPoint = ADD_MIN

                    #Special judge for wrong [max] before it
                    try:
                        if i >= 1 and newXMax[-1] == X[0] and newXMax[-1] < xm[i-1]:
                            #print newXMax[-1], xm[i-1], i
                            newXMax[-1] = xm[i-1]
                            newYMax[-1] = ym[i-1]
                    except:
                        pass
                else:
                    #Discard the [min]
                    j = j + 1
            else:
                #...max, [max], min...: Discard it
                i = i + 1
        elif lastPoint == ADD_MIN:
            if xm[i] < xn[j]:
                # print 'min, [max], min: ' + str(ym[i] - yn[j -1])
                if (ym[i] - yn[j -1]) > yTh:
                    #...min, [max], min...: Normal
                    # print 'add max:{}, x={}'.format(str(i), xm[i])
                    newXMax.append(xm[i])
                    newYMax.append(ym[i])
                    i = i + 1
                    lastPoint = ADD_MAX
                else:
                    #Discard the [max]
                    i = i + 1
            else:
                #...min, [min], max: Discard it
                # newXMax.append((xn[j] + xn[j-1]) / 2)
                # newYMax.append(ym[0])
                # newXMin.append(xn[j])
                # newYMin.append(yn[j])
                j = j + 1
                # lastPoint = ADD_MIN
    else:
        if i < len(_max) and lastPoint == ADD_MIN:
            #...min, [max].
            newXMax.append(xm[i])
            newYMax.append(ym[i])
            i = i + 1
            lastPoint = ADD_MAX
        elif j < len(_min) and lastPoint == ADD_MAX and (ym[i-1] - yn[j]) > yTh:
            #...max, [min] : Make a max after it
            newXMin.append(xn[j])
            newYMin.append(yn[j])
            newXMax.append(X[-1])
            newYMax.append(smoothY[-1])
            j = j + 1
            lastPoint = ADD_MAX

    # plot(newXMax, newYMax, 'r*', markersize = 10)
    # plot(newXMin, newYMin, 'k*', markersize = 10)
    # print fi

    meanDist = np.mean(newYMin)
    stdDist = np.std(newYMin)
    targetDist = meanDist + stdDist * paramD
    print 'Dist: mean:{}, std:{}'.format(str(meanDist),  str(stdDist))

    clickTimes = getClickTimes(X, smoothY, newXMin, targetDist)
    meanTime = np.mean(clickTimes)
    stdTime = np.std(clickTimes)
    targetTime = meanTime + stdTime * paramT
    print 'Time: mean:{}, std:{}'.format(str(meanTime), str(stdTime))

    return targetDist, targetTime * 1000
    def capture(self):
#Initialization of parameters
        contDist=0
        inrange=0
        red = (255,0,0,120)
        green = (0,255,0)
        blue = (0,0,255)
        darkBlue = (0,0,128)
        white = (255,255,255)
        black = (0,0,0)
        yellow = (255,255,0)
        gray= (205,200, 177)
        rpt=[ [0,0] for i in range(4)]
        X=0
        Y=0
        t=[]
        ir_x=[]
        ir_y=[]
        ir_s=[]
        t_i = time.time()  
        m = PyMouse()
        k = PyKeyboard()
        running=False
        buff=[[],[]]
        maxBuff=20
        buff[0]=q.miniQueue(maxBuff)
        buff[1]=q.miniQueue(maxBuff)
        gestures=[[6,2,8,2,8],[9,3,9]]
        gestures=[ [str(i2) for i2 in i]for i in gestures]
#recording flags        
        rec_flg =0
        flg=True
        mouse_flg=0
        click_flg=0
        doubleClick_flg=0
        drag_flg=0
        dragX=0
        dragY=0
        wait_flg=0
        timeHold=80 #in milliseconds                

#calibration
        mouseModeValue=80
        clickValue=80
        lagValue=100
        calibration=False
        mouseModeCalib=False
        startMouseModeCalib=False
        clickingCalib=False
        startClickModeCalib=False
        mouseModeCalibList=[]
        clickingCalibList=[]
        rightClickValue=180
#Gesture
        gestureRightThreshHold=1000
        gestureLeftThreshHold=300
        gestureDownThreshHold=720
        gestureUpThreshHold=400
        gesture_flg_UD=0
        gesture_flg_DU=0
        gesture_flg_LR=0
        gesture_flg_RL=0
#Check inrange
        LED1=[]
        LED2=[]
        LED3=[]
        LED4=[]
        rptList=[]
#Intialization of GUI
        os.environ['SDL_VIDEO_iWINDOW_POS'] = "%d,%d" % (0,0)
        pygame.init()

        myfont=pygame.font.SysFont("monospace",15)
        calibFont=pygame.font.SysFont(",monospace",20)
        #the default cursor
        DEFAULT_CURSOR = mouse.get_cursor()  
        #the hand cursor
        _HAND_CURSOR = (
        "     XX         ",
        "    X..X        ",
        "    X..X        ",
        "    X..X        ",
        "    X..XXXXX    ",
        "    X..X..X.XX  ",
        " XX X..X..X.X.X ",
        "X..XX.........X ",
        "X...X.........X ",
        " X.....X.X.X..X ",
        "  X....X.X.X..X ",
        "  X....X.X.X.X  ",
        "   X...X.X.X.X  ",
        "    X.......X   ",
        "     X....X.X   ",
        "     XXXXX XX   ")
        _HCURS, _HMASK = pygame.cursors.compile(_HAND_CURSOR, ".", "X")
        HAND_CURSOR = ((16, 16), (5, 1), _HCURS, _HMASK)

        infoObject = pygame.display.Info()
        width=infoObject.current_w
        height=infoObject.current_h
        screen=pygame.display.set_mode((width/2,height/2))
        #screen=pygame.display.set_mode((1200/3,760/3))
        


        print('press "c" to calibrate, then')
        print('press "r" to start recording')        
        
        while flg==True:            

            if calibration: #do calibration
                newList=self.findDegrees(rpt) #[(theta1,i1),(theta2,i2)....)]
                tipIndex, tipIndexAngle, kIndex,kIndexAngle=self.indexData(newList)
                tipThumb,tipThumbAngle,kThumb,kThumbAngle=self.thumbData(newList)
                averageX,averageY=self.centerFind(rpt)
#GUI section
                screen.fill(black)
#Drawing the Circles
                pygame.draw.circle(screen, yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
                pygame.draw.circle(screen, red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
                pygame.draw.circle(screen, green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
                pygame.draw.circle(screen, blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
                pygame.draw.circle(screen, white, (averageX/3,averageY/3),10)

                mouseModeDistance=fun.distanceVec(\
                [rpt[tipIndex][0]],\
                [rpt[tipIndex][1]],\
                [rpt[tipThumb][0]],\
                [rpt[tipThumb][1]])

                clickingDistance=fun.distanceVec(\
                [rpt[kIndex][0]],\
                [rpt[kIndex][1]],\
                [rpt[tipThumb][0]],\
                [rpt[tipThumb][1]])

                pygame.draw.rect(screen, gray, (0,5,500,60))

                if not (mouseModeCalib or startClickModeCalib or startMouseModeCalib or clickingCalib):
                    Calib1=calibFont.render("Press H to start",1,black)
                    screen.blit(Calib1,(0,15))
                if startMouseModeCalib and not mouseModeCalib:
                    Calib1=calibFont.render("Tap tip of thumb and tip of index",1,black)
                    screen.blit(Calib1,(0,15))
                    Calib2=calibFont.render("Press H to complete",1,black)
                    screen.blit(Calib2,(0,35))
                    pygame.draw.line(screen,white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),5 )
                    mouseModeCalibList.append(mouseModeDistance[0])                
                if startClickModeCalib and not clickingCalib:
                    Calib1=calibFont.render("Tap tip of thumb and knuckle of index",1,black)
                    screen.blit(Calib1,(0,15))
                    Calib2=calibFont.render("Press H to complete",1,black)
                    screen.blit(Calib2,(0,35))
                    pygame.draw.line(screen,white,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),(rpt[kIndex][0]/3,rpt[kIndex][1]/3),5 )
                    clickingCalibList.append(clickingDistance[0])                    
                if mouseModeCalib and clickingCalib:
                    calibrationDone=1
                    Calib1=calibFont.render("Calibration Completed",1,black)
                    screen.blit(Calib1,(0,15))
                    Calib2=calibFont.render("Press r to start recording",1,black)
                    screen.blit(Calib2,(0,35))
#Recording
            if rec_flg==1:
                newList=self.findDegrees(rpt) #[(theta1,i1),(theta2,i2)....)]
                tipIndex, tipIndexAngle, kIndex,kIndexAngle=self.indexData(newList)
                tipThumb,tipThumbAngle,kThumb,kThumbAngle=self.thumbData(newList)
                averageX,averageY=self.centerFind(rpt)
    #GUI section
                screen.fill(black)
    #Drawing the Circles
                pygame.draw.circle(screen, yellow, (rpt[tipIndex][0]/3,rpt[tipIndex][1]/3),10)
                pygame.draw.circle(screen, red, (rpt[kIndex][0]/3,rpt[kIndex][1]/3),10)
                pygame.draw.circle(screen, green, (rpt[tipThumb][0]/3,rpt[tipThumb][1]/3),10)
                pygame.draw.circle(screen, blue, (rpt[kThumb][0]/3,rpt[kThumb][1]/3),10)
                pygame.draw.circle(screen, white, (averageX/3,averageY/3),10)
    #Drawing the Information Text
                ITLabel=myfont.render(  "IndexTip"+" "+str(tipIndexAngle),1,(25,255,255))
                screen.blit(ITLabel,(rpt[tipIndex][0]/3,rpt[tipIndex][1]/3))
                IKLabel=myfont.render(  "IndexKnuck"+" "+str(kIndexAngle)   ,1,(255,255,255))
                screen.blit(IKLabel,(rpt[kIndex][0]/3,rpt[kIndex][1]/3))
                TTLabel=myfont.render(  "ThumbTip"+" "+str(tipThumbAngle)   ,1,(255,255,255))
                screen.blit(TTLabel,(rpt[tipThumb][0]/3,rpt[tipThumb][1]/3))
                TKLabel=myfont.render(  "ThumbKnuck"+" "+str(kThumbAngle)   ,1,(255,255,255))
                screen.blit(TKLabel,(rpt[kThumb][0]/3,rpt[kThumb][1]/3))
                speedLabel=myfont.render("Increase:z, Decrease:x",1,(255,255,255))
                screen.blit(speedLabel,(0,65))
                mouseLabel=myfont.render("Mouse:"+" "+str(mouseModeValue) ,1,(255,255,255))
                screen.blit(mouseLabel,(0,80))
                clickLabel=myfont.render("Click:"+" "+str(clickValue) ,1,(255,255,255))
                screen.blit(clickLabel,(0,95))
    #Mouse Events
        #Drawing the mode
                if mouse_flg:
                    MouseKeyboard=myfont.render( "Mouse mode",1,(255,255,255))
                else:
                    MouseKeyboard=myfont.render( "Keyboard mode",1,(255,255,255))
                screen.blit(MouseKeyboard,(0,50))
        #Distance for switching modes
                dista=fun.distanceVec(\
                [rpt[tipIndex][0]],\
                [rpt[tipIndex][1]],\
                [rpt[tipThumb][0]],\
                [rpt[tipThumb][1]])
        #Distance for clicking - thumb tip to index knuckle
                distClick=fun.distanceVec(\
                [rpt[kIndex][0]],\
                [rpt[kIndex][1]],\
                [rpt[tipThumb][0]],\
                [rpt[tipThumb][1]])
    #Switching Modes
                if 10<=dista[0]<=mouseModeValue and inrange==1:                
                    contDist+=1                        
                if contDist>=timeHold and mouse_flg==0 and drag_flg==0:
                    print('Mouse mode activated')
                    mouse_flg=1
                    contDist=0
                    mouse.set_cursor(*HAND_CURSOR)
                if contDist>=timeHold and mouse_flg==1 and drag_flg==0:
                    print('Mouse mode deactivated')
                    mouse_flg=0
                    contDist=0
                    mouse.set_cursor(*DEFAULT_CURSOR)
        #Adjusting MaxBuff with respect to thumbtip and index knuckle
                if mouse_flg:
                    a=40*clickValue
                    maxBuff=a/distClick[0]
                    if maxBuff<20:
                        maxBuff=20
                    elif maxBuff>40:
                        maxBuff=40
    #Clicking
                if distClick[0]<clickValue and inrange and mouse_flg and not click_flg:
                    click_flg=1
                    stime=time.time()
                    m.click(buff[0].mean(),buff[1].mean())
                    dragX, dragY=buff[0].mean(),buff[1].mean()
                    print('Click')
                    print distClick[0]
                if (click_flg and (time.time()-stime)*1000>=lagValue and not drag_flg): #so its been 1/2 second, 
                    if (distClick[0]>=clickValue): #if finger is up, then delete flag. Else 
                        click_flg=0
                        drag_flg=0
                        print("reset")
                        print distClick[0]
                    elif ((dragX-buff[0].mean()>5) or (dragY-buff[1].mean()>5)): #Drag situation
                        m.press(dragX,dragY)
                        drag_flg=1
                        print ("dragging")
                        print distClick[0]
                if drag_flg and distClick[0]>=int(1.2*clickValue): #released the drag
                    drag_flg=0
                    m.release(buff[0].mean(),buff[1].mean())
                    dragX,dragY=0,0
                    print("release drag")
                    print distClick[0]
    #right click needs improvement #tried using the thumb
                #if mouse_flg and rightClick[0]>rightClickValue: 
                #    m.click(buff[0].mean(),buff[1].mean(),2)
    #Gestures
        #The gesture bounds
            #    pygame.draw.line(screen,white, (gestureRightThreshHold/3,0),(gestureRightThreshHold/3,800))
            #    pygame.draw.line(screen,red, (gestureLeftThreshHold/3,0),(gestureLeftThreshHold/3,800))
            #    pygame.draw.line(screen,blue, (0,gestureDownThreshHold/3),(10000,gestureDownThreshHold/3))
            #    pygame.draw.line(screen,yellow, (0,gestureUpThreshHold/3),(10000,gestureUpThreshHold/3))
        #Swipe Right to Left
                if self.allAboveGestureRight(rpt,gestureRightThreshHold) and not gesture_flg_RL:
                    gestureTime=time.time()
                    gesture_flg_RL=1
                    print("ready to gesture")
                if gesture_flg_RL and (time.time()-gestureTime)<1:
                    if self.allAboveGestureLeft(rpt, gestureLeftThreshHold):
                        k.press_key(k.control_key)
                        k.press_key(k.alt_key)
                        k.press_key(k.left_key)
                        k.release_key(k.control_key)
                        k.release_key(k.alt_key)
                        k.release_key(k.left_key)
                        gesture_flg_RL=0
        #Swipe Left to Right
                if self.allAboveGestureLeft(rpt,gestureLeftThreshHold) and not gesture_flg_LR:
                    gestureTime=time.time()
                    gesture_flg_LR=1
                    print("ready to gesture")
                if gesture_flg_LR and (time.time()-gestureTime)<1:
                    if self.allAboveGestureRight(rpt, gestureRightThreshHold):
                        k.press_key(k.control_key)
                        k.press_key(k.alt_key)
                        k.press_key(k.right_key)
                        k.release_key(k.control_key)
                        k.release_key(k.alt_key)
                        k.release_key(k.right_key)
                        gesture_flg_LR=0
        #Swipe Down to Up
                if self.allAboveGestureDown(rpt,gestureDownThreshHold) and not gesture_flg_DU:
                    gestureTime=time.time()
                    gesture_flg_DU=1
                    print("ready to gesture")
                if gesture_flg_DU and (time.time()-gestureTime)<1:
                    if self.allAboveGestureUp(rpt, gestureUpThreshHold):
                        k.press_key(k.control_key)
                        k.press_key(k.alt_key)
                        k.press_key(k.up_key)
                        k.release_key(k.control_key)
                        k.release_key(k.alt_key)
                        k.release_key(k.up_key)
                        gesture_flg_DU=0
        #Swipe Up to Down
                if self.allAboveGestureUp(rpt,gestureUpThreshHold) and not gesture_flg_UD:
                    gestureTime=time.time()
                    gesture_flg_UD=1
                    print("ready to gesture")
                if gesture_flg_UD and (time.time()-gestureTime)<1:
                    if self.allAboveGestureDown(rpt, gestureDownThreshHold):
                        k.press_key(k.control_key)
                        k.press_key(k.alt_key)
                        k.press_key(k.down_key)
                        k.release_key(k.control_key)
                        k.release_key(k.alt_key)
                        k.release_key(k.down_key)
                        gesture_flg_UD=0
                if gesture_flg_RL and (time.time()-gestureTime)>=1:
                    gesture_flg_RL=0
                if gesture_flg_LR and (time.time()-gestureTime)>=1:
                    gesture_flg_LR=0
                if gesture_flg_UD and (time.time()-gestureTime)>=1:
                    gesture_flg_UD=0
                if gesture_flg_DU and (time.time()-gestureTime)>=1:
                    gesture_flg_DU=0
    #Capturing keyboard events          
            for event in pygame.event.get():
                if event.type==KEYDOWN:
                    if event.key==pygame.K_r: #start recording
                        rec_flg=1
                        calibration=False
                    elif event.key==pygame.K_c: #start calibration
                        calibration=1
                    elif event.key==pygame.K_s: #pauses the recording
                        rec_flg=False
                        break
                    elif event.key==pygame.K_q: #quits entirely
                        flg=False
                        break
                    if rec_flg: #if recording, can change the lag time
                        if event.key==pygame.K_z:
                            lagValue+=100
                        elif event.key==pygame.K_x:
                            lagValue-=100
                        

                #Mouse events for calibration mode
                    if calibration:
                        if not mouseModeCalib:
                            if not startMouseModeCalib and event.key==pygame.K_h:
                                startMouseModeCalib=True 
                            elif startMouseModeCalib and event.key==pygame.K_h:
                                mouseModeCalib=True
                                while min(mouseModeCalibList)<50:
                                    mouseModeCalibList.remove(min(mouseModeCalibList))
                                mouseModeValue=int(1.2*min(mouseModeCalibList))
                                mouseModeCalib=True
                        if mouseModeCalib:
                            if not startClickModeCalib and event.key==pygame.K_h:
                                startClickModeCalib=True
                            elif startClickModeCalib and event.key==pygame.K_h:
                                while min(clickingCalibList)<30:
                                    clickingCalibList.remove(min(clickingCalibList))
                                clickValue=int(1.2*min(clickingCalibList))
                                clickingCalib=True                            
                if event.type==QUIT:
                    flg=False
                    pygame.quit()
                    break
    #Capturing wii data                    
            messages = self.wii.get_mesg()    
            for mesg in messages:   # Loop through Wiimote Messages
                if mesg[0] == cwiid.MESG_IR: # If message is IR data
        #while recording data
                    if rec_flg == 1 or calibration:    # If recording
                        cont=-1
                        for s in mesg[1]:   # Loop through IR LED sources
                            cont+=1
                            if s:   # If a source exists
                                t.append(time.time()-t_i)
                                rpt[cont][0]=(1200-s['pos'][0])
                                rpt[cont][1]=s['pos'][1]
                    #Check inrange
                        newRpt=copy.deepcopy(rpt)
                        rptList.append(newRpt)
                        inrange, LED1,LED2,LED3,LED4=self.rangeChecker(rptList, LED1, LED2,LED3,LED4)
                #while in mouse mode
                    if mouse_flg==1:
                        X=rpt[tipIndex][0]
                        mouseX=(X-600)*width/400                    
                        Y=rpt[tipIndex][1]
                        mouseY=(Y-150)*height/290

                        """Currently we have the setting such that if there is a single LED that is out of range then
                        the mouse wont move. The problem with this is that the range of the mouse gets limited, and 
                        some places (such as corners) are difficult/impossible to click. If we eliminate the if statement
                        then this problem won't exist, but then it may start to recognize the knuckle LED as the tip and vice 
                        versa. So this is a give or take until we have a better filtering method."""

                        if inrange:
                            buff[0].put(mouseX)
                            buff[1].put(mouseY)
                            smoothX=np.mean(fun.smooth(buff[0].data, window_len=len(buff[0].data)))
                            smoothY=np.mean(fun.smooth(buff[1].data, window_len=len(buff[1].data)))
                            m.move(smoothX,smoothY)                    
                #I can also control using the wiimote
                elif mesg[0] == cwiid.MESG_BTN:  # If Message is Button data
                    if mesg[1] & cwiid.BTN_PLUS:    # Start Recording
                        rec_flg = 1
                        print "Recording..."
                    elif mesg[1] & cwiid.BTN_MINUS: # Stop Recording
                        flg=False
                        break
                pygame.display.update()         
        pygame.quit()