示例#1
0
class Gui(QtGui.QMainWindow):
    """ 
    Main GUI Class
    contains the main function and interfaces between 
    the GUI and functions
    """
    def __init__(self, parent=None):
        QtGui.QWidget.__init__(self, parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        """ Main Variables Using Other Classes"""
        self.rex = Rexarm()
        self.video = Video()
        self.statemachine = Statemachine()
        """ Other Variables """
        self.last_click = np.float32([0, 0])
        """ Customized events """
        self.flag_pickAndPlace = 0
        self.n_pickLocation = 2
        self.flag_event = 0
        """ Set GUI to track mouse """
        QtGui.QWidget.setMouseTracking(self, True)
        """ 
        GUI timer 
        Creates a timer and calls update_gui() function 
        according to the given time delay (27ms) 
        """
        self._timer = QtCore.QTimer(self)
        self._timer.timeout.connect(self.update_gui)
        self._timer.start(27)
        """
        Event timer
        Creates a timer and calls updateEvent() function 
        according to the given time delay (???ms) 
        """
        # Thread(target=self.event_one).start()
        # self._timer3 = QtCore.QTimer(self)
        # self._timer3.timeout.connect(self.event_one)
        # self._timer3.start(50)
        """ 
        LCM Arm Feedback timer
        Creates a timer to call LCM handler for Rexarm feedback
        """
        self._timer2 = QtCore.QTimer(self)
        self._timer2.timeout.connect(self.rex.get_feedback)
        self._timer2.start()
        """ 
        Connect Sliders to Function
        TODO: CONNECT THE OTHER 5 SLIDERS IMPLEMENTED IN THE GUI 
        """
        self.ui.sldrBase.valueChanged.connect(self.sliderChange)
        self.ui.sldrShoulder.valueChanged.connect(self.sliderChange)
        self.ui.sldrElbow.valueChanged.connect(self.sliderChange)
        self.ui.sldrWrist.valueChanged.connect(self.sliderChange)
        self.ui.sldrGrip1.valueChanged.connect(self.sliderChange)
        self.ui.sldrGrip2.valueChanged.connect(self.sliderChange)
        """ Initial command of the arm to home position"""
        self.sliderChange()
        """ Connect Buttons to Functions 
        TODO: NAME AND CONNECT BUTTONS AS NEEDED
        """
        self.ui.btnUser1.setText("Reset Location")
        self.ui.btnUser1.clicked.connect(self.reset)

        self.ui.btnUser2.setText("Play Event 1")
        self.ui.btnUser2.clicked.connect(self.event_one)
        self.ui.btnUser3.setText("Play Event 2")
        self.ui.btnUser3.clicked.connect(self.event_two)
        self.ui.btnUser4.setText("Play Event 3")
        self.ui.btnUser4.clicked.connect(self.event_three)
        self.ui.btnUser5.setText("Play Event 4")
        self.ui.btnUser5.clicked.connect(self.event_four)
        self.ui.btnUser6.setText("Play Event 5")
        self.ui.btnUser6.clicked.connect(self.event_five)
        # self.ui.btnUser6.setText("Test Arm Movement")
        # self.ui.btnUser6.clicked.connect(self.testArm)
        self.ui.btnUser7.setText("Click to grab and place")
        self.ui.btnUser7.clicked.connect(self.GrabAndPlace)
        # self.ui.btnUser8.setText("Teach and Repeat")
        # self.ui.btnUser8.clicked.connect(self.teach)
        # self.ui.btnUser9.setText("Record Arm Location")
        # self.ui.btnUser9.clicked.connect(self.record)
        # self.ui.btnUser10.setText("Play Arm Locations")
        # self.ui.btnUser10.clicked.connect(self.playback)
        self.ui.btnUser10.setText("Calibrate Camera")
        self.ui.btnUser10.clicked.connect(self.simple_cal)
        self.ui.btnUser11.setText("Configure Servos")
        self.ui.btnUser12.clicked.connect(self.rex.cfg_publish_default)
        self.ui.btnUser12.setText("Emergency Stop")
        self.ui.btnUser12.clicked.connect(self.estop)
        # self.ui.btnUser11.setText("Reach DW")
        # self.ui.btnUser11.clicked.connect(self.reach_down)

    def reset(self):
        self.statemachine.reset(self.rex)

    def event_one(self):
        eventMode = 1
        print('Start playing event 1:')
        self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def event_two(self):
        eventMode = 2
        print('Start playing event 2:')
        self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def event_three(self):
        eventMode = 3
        print('Start playing event 3:')
        self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def event_four(self):
        eventMode = 4
        print('Start playing event 4:')
        self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def event_five(self):
        eventMode = 5
        print('Start playing event 5:')
        self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def GrabAndPlace(self):
        eventMode = 6
        self.video.click2grab = 1
        self.video.mouse_click_id = 0
        # self.video.wld_coord = np.zeros((2,3))
        if self.video.click2grab == 2:
            print('Pick and place the following' + self.video.wld_coord)
            # self.statemachine.runStateMachine(self.video, self.rex, eventMode)

    def testArm(self):
        self.statemachine.testArmMove(self.ui, self.rex)

    def estop(self):
        self.statemachine.estop(self.rex)

    def update_gui(self):
        """ 
        update_gui function
        Continuously called by timer1 
        """
        """ Renders the video frames
            Displays a dummy image if no video available
            HINT: you can use the radio buttons to select different
            video sources like is done below
        """

        if (self.video.kinectConnected == 1):
            try:
                self.video.captureVideoFrame()
                self.video.captureDepthFrame()

            except TypeError:
                self.video.kinectConnected = 0
                self.video.loadVideoFrame()
                self.video.loadDepthFrame()

        if (self.ui.radioVideo.isChecked()):
            # self.ui.videoFrame
            self.ui.videoFrame.setPixmap(self.video.convertFrame())

        if (self.ui.radioDepth.isChecked()):
            self.ui.videoFrame.setPixmap(self.video.convertDepthFrame())
        """ 
        Update GUI Joint Coordinates Labels
        """
        self.ui.rdoutBaseJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[0] * R2D)))
        self.ui.rdoutShoulderJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[1] * R2D)))
        self.ui.rdoutElbowJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[2] * R2D)))
        self.ui.rdoutWristJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[3] * R2D)))
        self.ui.rdoutGrip1.setText(
            str("%.2f" % (self.rex.joint_angles_fb[4] * R2D)))
        self.ui.rdoutGrip2.setText(
            str("%.2f" % (self.rex.joint_angles_fb[5] * R2D)))
        """ 
        Mouse position presentation in GUI
        TODO: after implementing workspace calibration 
        display the world coordinates the mouse points to 
        in the RGB video image.
        """
        x = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).x()
        y = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).y()
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)):
            self.ui.rdoutMousePixels.setText("(-,-,-)")
            self.ui.rdoutMouseWorld.setText("(-,-,-)")
        else:
            x = x - MIN_X
            y = y - MIN_Y
            d = 0
            """ RGB, depth sensor registration"""
            if (self.ui.radioVideo.isChecked()):
                x_depth, y_depth = np.dot(rgb2depthAff, np.array([x, y, 1]).T)
                if y_depth >= 480 or x_depth >= 600 or y_depth <= 0 or x_depth <= 0:
                    self.ui.rdoutMousePixels.setText("(-,-,-)")
                else:
                    d = self.video.currentDepthFrame[int(y_depth)][int(
                        x_depth)]
            else:
                if y >= 480 or x >= 600 or y <= 0 or x <= 0:
                    self.ui.rdoutMousePixels.setText("(-,-,-)")
                else:
                    d = self.video.currentDepthFrame[int(y)][int(x)]
            """ Convert image coord  into world coord"""
            x_wld, y_wld, z_wld = 0, 0, 0
            x_wld, y_wld, z_wld = img2wld(x, y, d)
            self.ui.rdoutMousePixels.setText("(%.0f,%.0f,%.0f)" % (x, y, d))
            self.ui.rdoutMouseWorld.setText("(%.0f,%.0f,%.0f)" %
                                            (x_wld, y_wld, z_wld))
        """ 
        Updates status label when rexarm playback is been executed.
        This can be extended to include other appropriate messages
        """
        # if(self.rex.plan_status == 1):
        #     self.playback(maxMove)
        #     self.ui.rdoutStatus.setText("Playing Back - Waypoint %d"
        #                             %(self.rex.wpt_number))

    def playEvent(self):
        """
        Run state machine
        """
        if self.flag_event != 0:
            eventStatus = False  # True means assigned event is finished, otherwise False
            eventStatus = self.statemachine.runStateMachine(
                self.video, self.rex, self.flag_event)
        if eventStatus:
            self.flag_event = 0
            print('Event is finished, end of updating Gui')
        else:
            print('Event is not finished, end of updating Gui')

    def sliderChange(self):
        """ 
        Function to change the slider labels when sliders are moved
        and to command the arm to the given position
        """
        self.ui.rdoutBase.setText(str(self.ui.sldrBase.value()))
        self.ui.rdoutTorq.setText(str(self.ui.sldrMaxTorque.value()) + "%")
        self.ui.rdoutShoulder.setText(str(self.ui.sldrShoulder.value()))
        self.ui.rdoutElbow.setText(str(self.ui.sldrElbow.value()))
        self.ui.rdoutWrist.setText(str(self.ui.sldrWrist.value()))
        self.ui.rdoutGrip1.setText(str(self.ui.sldrGrip1.value()))
        self.ui.rdoutGrip2.setText(str(self.ui.sldrGrip2.value()))

        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = self.ui.sldrSpeed.value() / 100.0

        self.rex.joint_angles[0] = self.ui.sldrBase.value() * D2R
        self.rex.joint_angles[1] = self.ui.sldrShoulder.value() * D2R
        self.rex.joint_angles[2] = self.ui.sldrElbow.value() * D2R
        self.rex.joint_angles[3] = self.ui.sldrWrist.value() * D2R
        self.rex.joint_angles[4] = self.ui.sldrGrip1.value() * D2R
        self.rex.joint_angles[5] = self.ui.sldrGrip2.value() * D2R
        print('Joint angles from UI:')
        print(self.rex.joint_angles)
        self.rex.cmd_publish()
        print('Rexarm angle command published')
        # self.forward_kinematic()

    def mousePressEvent(self, QMouseEvent):
        """ 
        Function used to record mouse click positions for calibration 
        """
        """ Get mouse posiiton """
        x = QMouseEvent.x()
        y = QMouseEvent.y()
        """ If mouse position is not over the camera image ignore """
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)): return
        """ Change coordinates to image axis """
        self.last_click[0] = x - MIN_X
        self.last_click[1] = y - MIN_Y

        "Transform coordinates if cam calibration is not called"
        if (self.video.cal_flag == 0):
            if (self.ui.radioVideo.isChecked()):
                # Obtain x, y, z in RGB and depth sensor
                x = x - MIN_X
                y = y - MIN_Y
                x_depth, y_depth = np.dot(rgb2depthAff, np.array([x, y, 1]).T)
                if y_depth >= 480 or x_depth >= 600 or y_depth <= 0 or x_depth <= 0:
                    return
                z = self.video.currentDepthFrame[int(y_depth)][int(x_depth)]
                # Convert coord into world frame
                imgHom2Cam = cvtD2Z(z) * invIntrinsicMat
                camCoord = np.dot(imgHom2Cam, np.float32([x, y, 1]))
                camHomCoord = np.append(camCoord, [1])
                rw_v2 = np.dot(invExtrinsicMat, camHomCoord)
                x_wld, y_wld, z_wld = rw_v2[0] / rw_v2[3], rw_v2[1] / rw_v2[
                    3], rw_v2[2] / rw_v2[3]
                print('The mouse clicked world frame is: ')
                print(x_wld, y_wld, z_wld)
        """ get the RGB value at the cursor location"""
        # rgb_frame = cv2.cvtColor(self.video.currentVideoFrame, cv2.COLOR_RGB2BGR)
        # print(rgb_frame[y-MIN_Y][x-MIN_X])
        print('rgb: ')
        print(self.video.currentVideoFrame[y - MIN_Y][
            x - MIN_X])  # print RGB value at the click location
        print(self.video.currentDepthFrame[y - MIN_Y][
            x - MIN_X])  # print depth d at the click location
        """ Return clicked coords """
        # self.video.mouse_click_id = 0
        if (self.video.click2grab == 1):
            self.video.wld_coord[
                self.video.mouse_click_id] = x_wld, y_wld, z_wld
            self.video.mouse_click_id += 1

            if (self.video.mouse_click_id == 2):
                print('flag' + str(self.video.click2grab))
                self.video.click2grab = 2
                print('flag changed to ' + str(self.video.click2grab))
                self.statemachine.runStateMachine(self.video, self.rex, 6)
        """ If calibration is called to perform"""
        if (self.video.cal_flag == 1
            ):  # 0 - not performing calibration, 1 - performing calibraiton
            """ Save last mouse coordinate """
            self.video.mouse_coord[self.video.mouse_click_id] = [(x - MIN_X),
                                                                 (y - MIN_Y)]
            """ Update the number of used poitns for calibration """
            self.video.mouse_click_id += 1
            """ Update status label text """
            self.ui.rdoutStatus.setText("Affine Calibration: Click Point %d" %
                                        (self.video.mouse_click_id + 1))
            """ 
            If the number of click is equal to the expected number of points
            computes the affine calibration.
            
            TODO: LAB TASK: Change this code to use your workspace calibration routine
            and NOT the simple calibration function as is done now.
            """
            x = x - MIN_X
            y = y - MIN_Y

            if (self.ui.radioVideo.isChecked()):
                x, y = np.dot(rgb2depthAff, np.array([x, y, 1]).T)

            d = self.video.currentDepthFrame[int(y)][int(x)]

            depth = cvtD2Z(d)
            image_coord = np.append(
                self.video.mouse_coord[self.video.mouse_click_id - 1], [1])
            # projection_mat = np.array([[1,0,0,0],[0,1,0,0],[0,0,1,0]])

            camera_c = depth * np.dot(invIntrinsicMat, image_coord)

            print('Current cammera cord;')
            print(camera_c)
            self.video.camera_coord[self.video.mouse_click_id - 1] = camera_c

            print('Current #' + str(self.video.mouse_click_id))

            if (self.video.mouse_click_id == self.video.cal_points):
                """ 
                Update status of calibration flag and number of mouse
                clicks
                """
                self.video.mouse_click_id = 0

                wld2CamAffine = calAffine3D(self.video.real_coord,
                                            self.video.camera_coord)
                print(wld2CamAffine)

                wld2CamPadding = np.zeros((1, 4), dtype=float)
                wld2CamPadding[0, 3] = 1.

                self.video.extrinsicMat = np.concatenate(
                    (wld2CamAffine, wld2CamPadding), axis=0)
                print('Before thresholed extrinsic matrix:')
                print(self.video.extrinsicMat)
                threshold = 1e-7
                self.video.invExtrinsicMat = np.linalg.pinv(
                    self.video.extrinsicMat)

                img2CamPadding = np.zeros((1, 3), dtype=float)
                img2CamPadding[0, 2] = 1.
                self.video.imgHom2CamHom = np.concatenate(
                    (depth * invIntrinsicMat, img2CamPadding), axis=0)
                self.video.CamHom2WldHom = np.linalg.pinv(
                    self.video.extrinsicMat)
                self.video.aff_matrix = np.dot(self.video.CamHom2WldHom,
                                               self.video.imgHom2CamHom)

                print('Image coord is ')

                # self.video.aff_matrix = np.dot(extrinsicMat, np.dot(projection_mat.T,intrinsicMat))

                # self.video.aff_matrix = cv2.getAffineTransform(
                #                         self.video.mouse_coord,
                #                         self.video.real_coord)
                """ Updates Status Label to inform calibration is done """
                self.ui.rdoutStatus.setText("Waiting for input")
                print('**' * 10)
                # print(self.video.aff_matrix)
                print('Extrinsic matrix:')
                print(self.video.extrinsicMat)
                print('Inverse of Extrinsic matrix:')
                print(np.linalg.pinv(self.video.extrinsicMat))
                print('Intrinsic matrix:')
                print(intrinsicMat)
                print('Inverse Intrinsic matrix:')
                print(invIntrinsicMat)
                print('camera_c:')
                print(self.video.camera_coord)
                self.viedo.cal_flag = 0

        if (self.flag_pickAndPlace == 1):
            # if(self.video.mouse_click_id == self.video.cal_points):
            pos = np.array([x_wld, y_wld, z_wld])
            self.executePickAndPlace(pos)

    def simple_cal(self):
        """ 
        Function called when affine calibration button is called.
        Note it only chnages the flag to record the next mouse clicks
        and updates the status text label 
        """
        self.video.cal_flag = 1
        self.ui.rdoutStatus.setText("Affine Calibration: Click Point %d" %
                                    (self.video.mouse_click_id + 1))

    """ 
    Retired code, for testing kinematic 
    """
    # def inverse_kinematic_elbowup(self, target):
    # using the x,y,z,phi from self.getCurLoc()
    # to calculate theta0-3 and then go to the
    # targeted location
    # target = self.forward_kinematic()

    # angles = self.rex.rexarm_IK(target, 1) # 1-elbow up

    # if angles is None:
    #     print('ERROR: Can not reach the given location!')
    # else:
    #     print(np.asarray(angles) * R2D)
    #     return angles

    # def inverse_kinematic_elbowdown(self, target):
    #     # target = self.forward_kinematic()
    #     angles = self.rex.rexarm_IK(target, 0) # 0-elbow down

    #     if angles is None:
    #         print('ERROR: Can not reach the given location!')
    #     else:
    #         print(np.asarray(angles) * R2D)
    #         return angles

    # def reach_up(self, target):
    #     angles = self.inverse_kinematic_elbowup(target)
    #     print(angles)
    # self.rex.joint_angles[0], self.rex.joint_angles[1], \
    # self.rex.joint_angles[2], self.rex.joint_angles[3] = angles
    # # self.rex.cmd_publish()

    # def reach_down(self, target):
    #     angles = self.inverse_kinematic_elbowdown(target)
    #     self.rex.joint_angles[0], self.rex.joint_angles[1], \
    #     self.rex.joint_angles[2], self.rex.joint_angles[3] = angles
    #     self.rex.cmd_publish()

    # def forward_kinematic(self):
    # self.rex.torque_multiplier = 0
    # self.rex.cmd_publish()

    # all angles are in radius
    # theta1 = self.rex.joint_angles_fb[1] # squihoulder
    # theta2 = self.rex.joint_angles_fb[2] # elbow
    # theta3 = self.rex.joint_angles_fb[3] # wrist
    # # TODO: check link with i
    # a1 = 99 # link length
    # a2 = 99
    # a3 = 140

    # baseTheta = self.rex.joint_angles_fb[0] # base angle

    # dh_table = (theta1,theta2,theta3, a1, a2, a3, baseTheta)

    # x, y, z, phi = self.rex.rexarm_FK(dh_table, None)
    # self.ui.rdoutX.setText(str("%.2f mm" %x))
    # self.ui.rdoutY.setText(str("%.2f mm" %y))
    # self.ui.rdoutZ.setText(str("%.2f mm" %z))
    # self.ui.rdoutT.setText(str("%.2f deg" % (phi*R2D)))

    # return (x,y,z,phi)
    ""
    """
示例#2
0
class Gui(QtGui.QMainWindow):
    """ 
    Main GUI Class
    contains the main function and interfaces between 
    the GUI and functions
    """
    def __init__(self, parent=None):

        QtGui.QWidget.__init__(self, parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        """ Main Variables Using Other Classes"""
        self.rex = Rexarm()
        self.video = Video()
        self.statemachine = Statemachine()

        #self.video.extrinsic = np.array(self.video.getcaldata())
        #self.video.inv_extrinsic = np.array(np.linalg.inv(self.video.extrinsic))
        """ Other Variables """
        self.last_click = np.float32([0, 0])
        """ Set GUI to track mouse """
        QtGui.QWidget.setMouseTracking(self, True)
        self.statemachine.setme(self.ui, self.rex)
        """ 
        GUI timer 
        Creates a timer and calls update_gui() function 
        according to the given time delay (27ms) 
        """
        self._timer = QtCore.QTimer(self)
        self._timer.timeout.connect(self.update_gui)
        self._timer.start(27)
        """ 
        LCM Arm Feedback timer
        Creates a timer to call LCM handler for Rexarm feedback
        """
        self._timer2 = QtCore.QTimer(self)
        self._timer2.timeout.connect(self.rex.get_feedback)
        self._timer2.start()
        """
	Giri - Statemachine timer
	"""
        #self._timer3 = QtCore.QTimer(self)
        #self._timer3.timeout.connect(self.statemachine.timerCallback)
        #self._timer3.start(100)  # frequency of the timer is set by this. it is 100 ms now. However, the time.now can give microseconds precision not just millisecond
        """ 
        Connect Sliders to Function
        TODO: CONNECT THE OTHER 5 SLIDERS IMPLEMENTED IN THE GUI 
        """
        self.ui.sldrBase.valueChanged.connect(self.sliderBaseChange)
        self.ui.sldrShoulder.valueChanged.connect(
            self.sliderShoulderChange)  #Giri
        self.ui.sldrElbow.valueChanged.connect(self.sliderElbowChange)  #Giri
        self.ui.sldrWrist.valueChanged.connect(self.sliderWristChange)  #Giri
        self.ui.sldrGrip1.valueChanged.connect(self.sliderGrip1Change)  #Giri
        self.ui.sldrGrip2.valueChanged.connect(self.sliderGrip2Change)  #Giri
        self.ui.sldrMaxTorque.valueChanged.connect(
            self.sliderOtherChange)  #Giri
        self.ui.sldrSpeed.valueChanged.connect(self.sliderOtherChange)  #Giri
        if (self.rex.num_joints == 6):
            self.ui.sldrGrip1.setEnabled(True)  #Giri
            self.ui.sldrGrip2.setEnabled(True)  #Giri
        else:
            self.ui.sldrGrip1.setEnabled(False)  #Giri
            self.ui.sldrGrip2.setEnabled(False)  #Giri
        """ Initial command of the arm to home position"""
        self.sliderChange()
        self.reset()
        """ Connect Buttons to Functions 
        TODO: NAME AND CONNECT BUTTONS AS NEEDED
        """
        self.ui.btnUser1.setText("Configure Servos")
        self.ui.btnUser1.clicked.connect(self.btn1)  #Giri

        self.ui.btnUser2.setText("Depth and RGB Calibration")
        self.ui.btnUser2.clicked.connect(self.btn2)  #Giri
        self.ui.btnUser2.setEnabled(False)

        self.ui.btnUser3.setText("Extrinsic Calibration")  #Giri
        self.ui.btnUser3.clicked.connect(self.btn3)

        self.ui.btnUser4.setText("Block Detector")  #Giri
        self.ui.btnUser4.clicked.connect(self.btn4)

        self.ui.btnUser5.setText("Repeat")
        self.ui.btnUser5.clicked.connect(self.btn5)  #Giri

        self.ui.btnUser6.setText("Teach")
        self.ui.btnUser6.clicked.connect(self.teach)  #Giri

        self.ui.btnUser7.setText("Smooth Repeat")
        self.ui.btnUser7.clicked.connect(self.srepeat)  #Giri
        self.ui.btnUser7.setEnabled(False)  #Giri

        self.ui.btnUser8.setText("Pick Block")
        self.ui.btnUser8.clicked.connect(self.pick)

        self.ui.btnUser9.setText("Place Block")
        self.ui.btnUser9.clicked.connect(self.place)

        self.ui.btnUser10.setText("Reset")
        self.ui.btnUser10.clicked.connect(self.reset)  #Giri

        self.ui.btnUser11.setText("Enter Competition Mode")
        self.ui.btnUser11.clicked.connect(self.competition)  #Giri

        self.ui.btnUser12.setText("Emergency Stop!")
        self.ui.btnUser12.clicked.connect(self.estop)

    def estop(self):
        self.statemachine.estop(self.rex)

    def trim_angle(self, q, motorno=1):  #for 1,2,3
        if (motorno > 2):
            max_angle = 300
        else:
            max_angle = 360

        if (q > max_angle or q < -1 * max_angle):
            new_q = 0
            print "trimmed: " + str(q) + " motor no: " + str(motorno)
        elif (q > 180):  #181 --> -179
            new_q = q - 360
            print "trimmed: " + str(q) + " motor no: " + str(motorno)
        elif (q < -180):
            new_q = 360 - q
            print "trimmed: " + str(q) + " motor no: " + str(motorno)
        else:
            new_q = q
        return new_q

    def shortorientation(self, w_angle):
        if (w_angle > 90):
            sw_angle = w_angle - 90
        elif (w_angle < -90):
            sw_angle = w_angle + 90
        else:
            sw_angle = w_angle
        return sw_angle

    def update_gui(self):
        """ 
        update_gui function
        Continuously called by timer1 
        """
        """ Renders the video frames
            Displays a dummy image if no video available
            HINT: you can use the radio buttons to select different
            video sources like is done below
        """
        color = self.statemachine.timerCallback()
        if (color == "Decluttered"):
            self.declutter()
        elif (color != "none"):
            self.getangles(color)
        if (self.video.kinectConnected == 1):
            try:
                self.video.captureVideoFrame()
                self.video.captureDepthFrame()
            except TypeError:
                self.video.kinectConnected = 0
                self.video.loadVideoFrame()
                self.video.loadDepthFrame()

        if (self.ui.radioVideo.isChecked()):
            self.ui.videoFrame.setPixmap(self.video.convertFrame())

        if (self.ui.radioDepth.isChecked()):
            self.ui.videoFrame.setPixmap(self.video.convertDepthFrame())
        """ 
        Update GUI Joint Coordinates Labels
        """
        self.ui.rdoutBaseJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[0] * R2D)))
        self.ui.rdoutShoulderJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[1] * R2D)))
        self.ui.rdoutElbowJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[2] * R2D)))
        self.ui.rdoutWristJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[3] * R2D)))
        self.ui.rdoutGrip1.setText(str(self.ui.sldrGrip1.value()))  #Giri
        self.ui.rdoutGrip2.setText(str(self.ui.sldrGrip2.value()))  #Giri
        """
        Update End Effector Coordinates Kevin
	"""
        endCoord = forwardKinematics(self.rex.joint_angles_fb[0] * R2D,
                                     self.rex.joint_angles_fb[1] * R2D,
                                     self.rex.joint_angles_fb[2] * R2D,
                                     self.rex.joint_angles_fb[3] * R2D)
        self.ui.rdoutX.setText(str("%2f" % (round(endCoord[0], 2))))
        self.ui.rdoutY.setText(str("%2f" % (round(endCoord[1], 2))))
        self.ui.rdoutZ.setText(str("%2f" % (round(endCoord[2], 2))))
        self.ui.rdoutT.setText(str("%2f" % (round(endCoord[3], 2))))
        """ 
        Mouse position presentation in GUI
        TODO: after implementing workspace calibration 
        display the world coordinates the mouse points to 
        in the RGB video image.
        """
        x = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).x()
        y = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).y()
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)):
            self.ui.rdoutMousePixels.setText("(-,-,-)")
            self.ui.rdoutMouseWorld.setText("(-,-,-)")
        else:
            x = x - MIN_X
            y = y - MIN_Y
            z = self.video.currentDepthFrame[y][x]
            self.ui.rdoutMousePixels.setText("(%.0f,%.0f,%.0f)" % (x, y, z))
            if (self.video.cal_flag == 2):
                M = self.video.aff_matrix
                v = np.float32([x, y, 1])
                rw = np.dot(M, v)
                self.ui.rdoutMouseWorld.setText("(%.0f,%.0f,0)" %
                                                (rw[0], rw[1]))
            elif (self.video.extrinsic_cal_flag == 2):  #Josh
                xy_in_rgb = np.float32([[[x, y]]])  #Josh
                xy_in_rgb_homogenous = np.array([x, y, 1.0])
                self.video.rgb2dep_aff_matrix = np.array(
                    [[1.09403672e0, 3.44369111e-3, -1.62867595e0],
                     [-2.41966785e-3, 1.07534829e0, -2.69041712e1],
                     [0., 0., 1.]])  #Josh
                xy_in_dep = np.dot(self.video.rgb2dep_aff_matrix,
                                   xy_in_rgb_homogenous)  #Josh
                x_dep = int(xy_in_dep[0])  #Josh
                y_dep = int(xy_in_dep[1])  #Josh
                d = self.video.currentDepthFrame[y_dep][x_dep]
                Z = -1
                #Z = 0.1236*np.tan(d/2842.5 + 1.1863)*1000
                for idx in range(len(self.video.levels_mm)):
                    if d <= self.video.levels_upper_d[
                            idx] and d >= self.video.levels_lower_d[idx]:
                        Z = 941 - self.video.levels_mm[idx]
                        break
                camera_coord = Z * cv2.undistortPoints(
                    xy_in_rgb, self.video.intrinsic,
                    self.video.distortion_array)
                camera_coord_homogenous = np.transpose(
                    np.append(camera_coord, [Z, 1.0]))
                world_coord_homogenous = np.dot(self.video.extrinsic,
                                                camera_coord_homogenous)

                #camera_coord = Z*np.dot(self.video.inv_intrinsic,xy_in_rgb)
                #camera_coord_homo = np.concatenate((camera_coord,[1.]),axis = 0)
                #world_coord_homo = np.dot(self.video.extrinsic,camera_coord_homo)
                self.ui.rdoutMouseWorld.setText(
                    "(%.0f,%.0f,%.0f)" %
                    (world_coord_homogenous[0], world_coord_homogenous[1],
                     world_coord_homogenous[2]))
#self.ui.rdoutMouseWorld.setText("(%.0f,%.0f,%.0f)" % (x_dep,y_dep,d))
            else:
                self.ui.rdoutMouseWorld.setText("(-,-,-)")
        """ 
        Updates status label when rexarm playback is been executed.
        This can be extended to include other appropriate messages
        """
        if (self.rex.plan_status == 1):
            self.ui.rdoutStatus.setText("Playing Back - Waypoint %d" %
                                        (self.rex.wpt_number + 1))
        self.ui.rdoutStatus.setText(
            self.statemachine.getmestatus(combined=True))

    def sliderChange(self):
        """ 
        Function to change the slider labels when sliders are moved
        and to command the arm to the given position
        """
        self.ui.rdoutBase.setText(str(self.ui.sldrBase.value()))
        self.ui.rdoutTorq.setText(str(self.ui.sldrMaxTorque.value()) + "%")
        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = [self.ui.sldrSpeed.value() / 100.0
                                     ] * self.rex.num_joints
        self.rex.joint_angles[0] = self.ui.sldrBase.value() * D2R
        self.rex.joint_angles[1] = self.ui.sldrShoulder.value() * D2R
        self.rex.joint_angles[2] = self.ui.sldrElbow.value() * D2R
        self.rex.joint_angles[3] = self.ui.sldrWrist.value() * D2R
        if (self.rex.num_joints == 6):
            self.rex.joint_angles[4] = self.ui.sldrWrist.value() * D2R
            self.rex.joint_angles[5] = self.ui.sldrWrist.value() * D2R

        self.rex.cmd_publish()

    def sliderinitChange(self):  #Giri
        """ 
        Function to change the slider labels when sliders are moved
        and to command the arm to the given position
        """
        self.ui.rdoutBase.setText(str(self.ui.sldrBase.value()))
        self.ui.rdoutShoulder.setText(str(self.ui.sldrShoulder.value()))  #Giri
        self.ui.rdoutElbow.setText(str(self.ui.sldrElbow.value()))  #Giri
        self.ui.rdoutWrist.setText(str(self.ui.sldrWrist.value()))  #Giri
        self.ui.rdoutGrip1.setText(str(self.ui.sldrGrip1.value()))  #Giri
        self.ui.rdoutGrip2.setText(str(self.ui.sldrGrip2.value()))  #Giri

        self.ui.rdoutTorq.setText(str(self.ui.sldrMaxTorque.value()) + "%")
        self.ui.rdoutSpeed.setText(str(self.ui.sldrSpeed.value()) + "%")

        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = [self.ui.sldrSpeed.value() / 100.0
                                     ] * self.rex.num_joints
        #Giri{
        self.rex.joint_angles[0] = self.ui.sldrBase.value() * D2R
        self.rex.joint_angles[1] = self.ui.sldrShoulder.value() * D2R
        self.rex.joint_angles[2] = self.ui.sldrElbow.value() * D2R
        self.rex.joint_angles[3] = self.ui.sldrWrist.value() * D2R
        if (len(self.rex.joint_angles) == 6):
            self.rex.joint_angles[4] = self.ui.sldrGrip1.value() * D2R
            self.rex.joint_angles[5] = self.ui.sldrGrip2.value() * D2R
        #Giri}
        self.rex.cmd_publish()

    #From Giri: I implemented all these function in a one function above, incase if the response time is bad, we can go back to the below five separate functions

    def sliderOtherChange(self):  #Giri
        self.ui.rdoutTorq.setText(str(self.ui.sldrMaxTorque.value()) + "%")
        self.ui.rdoutSpeed.setText(str(self.ui.sldrSpeed.value()) + "%")
        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = [self.ui.sldrSpeed.value() / 100.0
                                     ] * self.rex.num_joints
        self.rex.cmd_publish()

    def sliderBaseChange(self):  #Kevin
        self.ui.rdoutBase.setText(str(self.ui.sldrBase.value()))
        self.rex.joint_angles[0] = self.ui.sldrBase.value() * D2R
        self.rex.cmd_publish()

    def sliderShoulderChange(self):  #Kevin
        self.ui.rdoutShoulder.setText(str(self.ui.sldrShoulder.value()))
        self.rex.joint_angles[1] = self.ui.sldrShoulder.value() * D2R
        self.rex.cmd_publish()

    def sliderElbowChange(self):  #Kevin
        self.ui.rdoutElbow.setText(str(self.ui.sldrElbow.value()))
        self.rex.joint_angles[2] = self.ui.sldrElbow.value() * D2R
        self.rex.cmd_publish()

    def sliderWristChange(self):  #Kevin
        self.ui.rdoutWrist.setText(str(self.ui.sldrWrist.value()))
        self.rex.joint_angles[3] = self.ui.sldrWrist.value() * D2R
        self.rex.cmd_publish()

    def sliderGrip1Change(self):  #Giri
        self.ui.rdoutGrip1.setText(str(self.ui.sldrGrip1.value()))
        self.rex.joint_angles[4] = self.ui.sldrGrip1.value() * D2R
        self.rex.cmd_publish()

    def sliderGrip2Change(self):  #Giri
        self.ui.rdoutGrip2.setText(str(self.ui.sldrGrip2.value()))
        self.rex.joint_angles[5] = self.ui.sldrGrip2.value() * D2R
        self.rex.cmd_publish()

    def mousePressEvent(self, QMouseEvent):
        """ 
        Function used to record mouse click positions for calibration 
        """
        """ Get mouse posiiton """
        x = QMouseEvent.x()
        y = QMouseEvent.y()
        """ If mouse position is not over the camera image ignore """
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)): return
        """ Change coordinates to image axis """
        self.last_click[0] = x - MIN_X
        self.last_click[1] = y - MIN_Y
        """ If calibration has been performed """
        if (self.video.cal_flag == 1):
            """ Save last mouse coordinate """
            self.video.mouse_coord[self.video.mouse_click_id] = [(x - MIN_X),
                                                                 (y - MIN_Y)]
            """ Update the number of used poitns for calibration """
            self.video.mouse_click_id += 1
            """ Update status label text """
            self.ui.rdoutStatus.setText(
                "Affine Calibration: Click Point %d is recorded, please select Click Point %d"
                % (self.video.mouse_click_id, self.video.mouse_click_id + 1))
            """ 
            If the number of click is equal to the expected number of points
            computes the affine calibration.
            
            LAB TASK: Change this code to use your workspace calibration routine
            and NOT the simple calibration function as is done now.
            """
            if (self.video.mouse_click_id == self.video.cal_points):
                """ 
                Update status of calibration flag and number of mouse
                clicks
                """
                self.video.cal_flag = 2
                self.video.mouse_click_id = 0
                """ Perform affine calibration with OpenCV """
                self.video.aff_matrix = cv2.getAffineTransform(
                    self.video.mouse_coord, self.video.real_coord)
                #print self.video.mouse_coord #Josh
                #print self.video.real_coord #Josh
                """ Updates Status Label to inform calibration is done """
                self.ui.rdoutStatus.setText("Waiting for input")
        if (self.video.dep2rgb_aff_flag == 1):  #Josh
            if (self.video.mouse_click_id < self.video.rgb_pts_num):
                self.video.rgb_coord[self.video.mouse_click_id * 2] = x - MIN_X
                self.video.rgb_coord[self.video.mouse_click_id * 2 +
                                     1] = y - MIN_Y
            else:
                self.video.dep_coord[2 * (self.video.mouse_click_id -
                                          self.video.rgb_pts_num)] = [
                                              (x - MIN_X), (y - MIN_Y), 1, 0,
                                              0, 0
                                          ]
                self.video.dep_coord[
                    2 * (self.video.mouse_click_id - self.video.rgb_pts_num) +
                    1] = [0, 0, 0, (x - MIN_X), (y - MIN_Y), 1]
            self.video.mouse_click_id += 1
            print self.video.mouse_click_id
            if (self.video.mouse_click_id == self.video.dep_pts_num):
                self.video.dep2rgb_aff_flag = 2
                self.video.mouse_click_id = 0
                A = self.video.dep_coord
                A_trans = np.transpose(A)
                A_AT = np.dot(A_trans, A)
                A_AT_inv = np.linalg.inv(A_AT)
                A_all = np.dot(A_AT_inv, A_trans)
                b = np.transpose(self.video.rgb_coord)
                tmp_dep2rgb_aff_array = np.dot(A_all, b)
                upper_dep2rgb_aff_matrix = np.reshape(tmp_dep2rgb_aff_array,
                                                      (2, 3))
                lower_dep2rgb_aff_matrix = np.array([[0, 0, 1]])
                self.video.dep2rgb_aff_matrix = np.concatenate(
                    (upper_dep2rgb_aff_matrix, lower_dep2rgb_aff_matrix),
                    axis=0)
                self.video.rgb2dep_aff_matrix = np.linalg.inv(
                    self.video.dep2rgb_aff_matrix)
                print self.video.rgb2dep_aff_matrix
        if (self.video.extrinsic_cal_flag == 1):  #Josh
            self.video.extrinsic_cal_pixel_coord[self.video.mouse_click_id] = [
                (x - MIN_X), (y - MIN_Y)
            ]
            self.video.mouse_click_id += 1
            if (self.video.mouse_click_id == 3):
                """
		retval, rvec, tvec = cv2.solvePnP(self.video.extrinsic_cal_world_coord, self.video.extrinsic_cal_pixel_coord, self.video.intrinsic, self.video.distortion_array)
		#print self.video.extrinsic_cal_pixel_coord, self.video.extrinsic_cal_world_coord		
		self.video.mouse_click_id = 0
		R,tmp = cv2.Rodrigues(rvec)
		tmp = np.concatenate((np.transpose(R),-1.0*tvec),axis=1)
		self.video.extrinsic = np.concatenate((tmp, np.float32([[0., 0., 0., 1.]])),axis = 0)
		print self.video.extrinsic
		self.video.inv_extrinsic = np.linalg.inv(self.video.extrinsic)
		self.video.extrinsic_cal_flag = 2
		"""
                self.video.mouse_click_id = 0
                tmp_coord = np.float32(
                    [[self.video.extrinsic_cal_pixel_coord[0]],
                     [self.video.extrinsic_cal_pixel_coord[1]],
                     [self.video.extrinsic_cal_pixel_coord[2]]])
                camera_coord = 941.0 * cv2.undistortPoints(
                    tmp_coord, self.video.intrinsic,
                    self.video.distortion_array)
                rot_rad = -1.0 * np.arctan2(
                    (camera_coord[1][0][1] - camera_coord[0][0][1]),
                    (camera_coord[1][0][0] - camera_coord[0][0][0]))
                if rot_rad < 0:
                    z_rot = rot_rad + 0.5 * np.pi
                else:
                    z_rot = 0.5 * np.pi - rot_rad
                R_T = np.array([[np.cos(z_rot),
                                 np.sin(z_rot), 0.0],
                                [np.sin(z_rot), -1.0 * np.cos(z_rot), 0.0],
                                [0.0, 0.0, -1.0]])
                R = np.transpose(R_T)
                T = np.float32([
                    (camera_coord[0][0][0] + camera_coord[2][0][0]) / 2,
                    (camera_coord[0][0][1] + camera_coord[2][0][1]) / 2, 941
                ])
                T = np.transpose(np.array([-1.0 * np.dot(R, T)]))
                tmp = np.concatenate((R, T), axis=1)
                self.video.extrinsic = np.concatenate(
                    (tmp, np.float32([[0., 0., 0., 1.]])), axis=0)
                #self.video.writecaldata(self.video.extrinsic)
                self.video.inv_extrinsic = np.linalg.inv(self.video.extrinsic)
                print "Extrinsic Matrix = ", self.video.extrinsic
                self.video.extrinsic_cal_flag = 2

    def deprgb_cali(self):  #Josh
        self.video.dep2rgb_aff_flag = 1
        self.ui.rdoutStatus.setText(
            "Start camera calibration by clicking mouse to select Click Point 1"
        )

    def extrinsic_cali(self):  #Josh

        alpha = 525.91386088
        u0 = 312.41480582
        beta = 526.52064559
        v0 = 247.78962122
        matrix1 = np.float32([[1 / alpha, 0., 0.], [0., 1 / beta, 0.],
                              [0., 0., 1.0]])
        matrix2 = np.float32([[1.0, 0., -u0], [0., 1., -v0], [0., 0., 1.0]])
        self.video.inv_intrinsic = np.dot(matrix1, matrix2)
        self.video.intrinsic = np.float32([[alpha, 0., u0], [0., beta, v0],
                                           [0., 0., 1.]])
        self.video.distortion_array = np.float32([
            2.45479098e-01, -8.27672136e-01, 1.05870966e-03, -3.01947277e-03,
            1.08683931e+00
        ])
        self.video.extrinsic_cal_flag = 1

    def teach(self):  #Giri
        [x, y, z, orientation] = [0, 14, 1, 180.0]
        q = [0.0, 0.0, 0.0, 0.0]
        [q[0], q[1], q[2], q[3]] = inverseKinematics(x, y, z, orientation)
        print q
        self.rex.joint_angles[0] = q[0] * D2R
        self.rex.joint_angles[1] = q[1] * D2R
        self.rex.joint_angles[2] = q[2] * D2R
        self.rex.joint_angles[3] = q[3] * D2R
        self.rex.cmd_publish()

    def repeat(self):  #Giri
        if (self.ui.btnUser7.text() == "Repeat"):
            self.statemachine.init(self.ui, self.rex, "Initialising")
        else:
            self.statemachine.repeat(self.ui, self.rex, False)  #Giri

    def srepeat(self):  #Giri
        self.statemachine.init(self.ui, self.rex, "initialising")

    def reset(self, hold=False):
        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = [self.ui.sldrSpeed.value() / 100.0
                                     ] * self.rex.num_joints

        self.rex.joint_angles[0] = 0.0
        self.rex.joint_angles[1] = 0.0
        self.rex.joint_angles[2] = 0.0
        self.rex.joint_angles[3] = 0.0
        if (len(self.rex.joint_angles) == 6 and hold == False):
            self.rex.joint_angles[4] = 0.0
            self.rex.joint_angles[5] = 95 * D2R
        elif (len(self.rex.joint_angles) == 6 and hold == True):
            self.rex.joint_angles[4] = 0.0
            self.rex.joint_angles[5] = -95 * D2R

        self.rex.cmd_publish()

    def getheight(self, angle, action="picking"):
        placement_offset = 1  # in cm
        if (angle - 90 < 0.1):
            gripping_height = 3
        elif (angle - 120 < 0.1):
            gripping_height = 2

        else:
            if (action == "picking"):
                gripping_height = 0.05
            elif (action == "placing"):
                gripping_height = 0.05 + placement_offset
        return gripping_height

    def getIK(self, endCoord, angle=0, action="picking"):
        intermediate_height = 4
        intermediate_angles = [0.0] * 6
        angles = [0.0] * 6
        [angles[0], angles[1], angles[2], angles[3]] = inverseKinematics(
            endCoord[0], endCoord[1],
            endCoord[2] + self.getheight(endCoord[3], action), endCoord[3])

        if (self.checkfound(angles)
            ):  # or self.checkfound(intermediate_angles,True)):
            if (abs(endCoord[3] - 90) < 0.1):
                endCoord[3] = 180
                [angles[0], angles[1], angles[2],
                 angles[3]] = inverseKinematics(
                     endCoord[0], endCoord[1],
                     endCoord[2] + self.getheight(endCoord[3], action),
                     endCoord[3])

            else:
                endCoord[3] = 90
                [angles[0], angles[1], angles[2],
                 angles[3]] = inverseKinematics(
                     endCoord[0], endCoord[1],
                     endCoord[2] + self.getheight(endCoord[3], action),
                     endCoord[3])
        if (self.checkfound(angles)
            ):  # or self.checkfound(intermediate_angles,True)):
            endCoord[3] = 120

            [angles[0], angles[1], angles[2], angles[3]] = inverseKinematics(
                endCoord[0] - 1, endCoord[1] + 1,
                endCoord[2] + self.getheight(endCoord[3], action), endCoord[3])

        if (endCoord[3] == 120):
            [
                intermediate_angles[0], intermediate_angles[1],
                intermediate_angles[2], intermediate_angles[3]
            ] = inverseKinematics(endCoord[0] - 1, endCoord[1] + 1,
                                  endCoord[2] + intermediate_height,
                                  endCoord[3])  # assuming it is reachable
            if (self.checkfound(intermediate_angles)):
                [
                    intermediate_angles[0], intermediate_angles[1],
                    intermediate_angles[2], intermediate_angles[3]
                ] = inverseKinematics(endCoord[0] - 1, endCoord[1] + 1,
                                      endCoord[2] + intermediate_height,
                                      90)  # assuming it is reachable

        if (endCoord[3] == 180):
            [
                intermediate_angles[0], intermediate_angles[1],
                intermediate_angles[2], intermediate_angles[3]
            ] = inverseKinematics(endCoord[0], endCoord[1],
                                  endCoord[2] + intermediate_height +
                                  self.getheight(endCoord[3], action),
                                  endCoord[3])  # assuming it is reachable

            print self.trim_angle(orientation(angles[0], angle * R2D))
            if (action == "placing"):
                angles[4] = 90 + angles[0]
            else:
                angles[4] = -1 * self.trim_angle(
                    orientation(angles[0], angle * R2D))

        else:
            angles[4] = 0
        self.checkfound(intermediate_angles, True)

        intermediate_angles[4] = angles[4]
        angles[5] = endCoord[3]
        intermediate_angles[5] = endCoord[3]
        if (self.checkfound(angles)
                and self.checkfound(intermediate_angles, False)):
            print "Cannot be picked"
        return [angles, intermediate_angles]

    def checkfound(self, angles, i=False):
        if (i):
            if (angles[0] == 0 and angles[1] == 0 and angles[2] == 0
                    and angles[3] == 0):
                print "intermediate angles not found"
                return True
        else:
            if (angles[0] == 0 and angles[1] == 0 and angles[2] == 0
                    and angles[3] == 0):
                print "final angles not found"
                return True
        return False

    def printfk(self, angles):
        temp = forwardKinematics(angles[0], angles[1], angles[2], angles[3])
        print "Forward Kinematics output =  ", str(temp[0]), str(temp[1]), str(
            temp[2]), str(temp[3])

    def roundoff(self, angles):
        angles[0] = round(self.trim_angle(angles[0]), 2)
        angles[1] = round(self.trim_angle(angles[1]), 2)
        angles[2] = round(self.trim_angle(angles[2]), 2)
        angles[3] = round(self.trim_angle(angles[3]), 2)
        angles[4] = round(self.trim_angle(angles[4]),
                          2)  #Dont alter these two angles
        angles[5] = round(self.trim_angle(angles[5]), 2)
        return angles

    def pick(self):  #Josh
        global putx, puty, putz, putangle
        blockx, blocky, blockz, angle = self.get_color_block_world_coord(
            'blue')
        #self.statemachine.setorientation(angle*R2D)
        [putx, puty, putz, putangle] = [-1 * blockx, blocky, blockz, angle]
        #endCoord = [20.2, -16.1, 4.4,90]
        #endCoord = [14.3, 19.4, 4.4,90]
        endCoord = [
            blockx / 10, blocky / 10, (blockz) / 10, gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, angle)
        if (debug):
            print "Block detector output = ", [
                blockx, blocky, blockz, angle * R2D
            ]
            print "Input to Inverse Kinematics:", endCoord
            print "Output of Inverse Kinematics: ", angles
            self.printfk(angles)

        angles = self.roundoff(angles)
        intermediate_angles = self.roundoff(intermediate_angles)
        self.statemachine.setq(angles, intermediate_angles)
        self.statemachine.setmystatus(
            "testing", "picking", "picking")  #mode="testing",action="picking")
        #self.statemachine.hold(self.ui,self.rex, angles,"picking")
    def place(self):

        endCoord = [putx / 10, puty / 10, (putz) / 10, gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, putangle)

        #wangle = (self.shortorientation((putangle)*R2D-45)-self.trim_angle(angles[0]))#self.shortorientation((angle)*R2D+45)
        #self.statemachine.set_orientations(wangle,2)
        if (debug):
            print "Placing according to Block detector output = ", [
                putx, puty, putz, putangle * R2D
            ]
            print "Input to Inverse Kinematics:", endCoord
            self.printfk(angles)

        angles = self.roundoff(angles)
        intermediate_angles = self.roundoff(intermediate_angles)

        self.statemachine.setq(angles, intermediate_angles)
        self.statemachine.setmystatus(
            "testing", "placing", "placing")  #mode="testing",action="placing")


#	self.statemachine.hold(self.ui,self.rex, angles,"placing")

    def get_color_block_world_coord(self, color):  #Josh
        self.video.blockDetector()
        [blockx_rgb, blocky_rgb, angle] = self.video.color_detection(color)
        if (blockx_rgb == 0 and blocky_rgb == 0 and angle == 0):
            return 0, 0, 0, 0
        xy_in_rgb = np.float32([[[blocky_rgb, blockx_rgb]]])  #Josh
        xy_in_rgb_homogenous = np.array([blocky_rgb, blockx_rgb, 1.0])
        xy_in_dep = np.dot(self.video.rgb2dep_aff_matrix,
                           xy_in_rgb_homogenous)  #Josh
        x_dep = int(xy_in_dep[0])  #Josh
        y_dep = int(xy_in_dep[1])  #Josh
        d = self.video.currentDepthFrame[y_dep][x_dep]
        Z = -1
        for idx in range(len(self.video.levels_mm)):
            if d <= self.video.levels_upper_d[
                    idx] and d >= self.video.levels_lower_d[idx]:
                Z = 941 - self.video.levels_mm[idx]
                break
        camera_coord = Z * cv2.undistortPoints(xy_in_rgb, self.video.intrinsic,
                                               self.video.distortion_array)
        camera_coord_homogenous = np.transpose(
            np.append(camera_coord, [Z, 1.0]))
        world_coord_homogenous = np.dot(self.video.extrinsic,
                                        camera_coord_homogenous)
        theta = np.arccos(self.video.extrinsic[0, 0])
        angle = angle - theta
        return world_coord_homogenous[0], world_coord_homogenous[
            1], world_coord_homogenous[2], angle

    def getangles(self, color):
        blockx, blocky, blockz, angle = self.get_color_block_world_coord(color)
        if (blockx == 0 and blocky == 0 and blockx == 0 and angle == 0):
            angles = [0] * 6
            intermediate_angles = [0] * 6
            print "not found"
            self.statemachine.set_comp5_status("not found")

        else:
            [
                current_mode, current_action, current_movement,
                current_motorstate
            ] = self.statemachine.getmestatus()
            endCoord = [(blockx) / 10, (blocky) / 10, (blockz) / 10,
                        gripper_orientation]

            [angles, intermediate_angles] = self.getIK(endCoord, angle,
                                                       "picking")

            angles = self.roundoff(angles)
            intermediate_angles = self.roundoff(intermediate_angles)
            print "found"
            self.statemachine.set_comp5_status("found")
        self.statemachine.setq(angles, intermediate_angles)

    def generatecomp2(self):
        print "Entered generatecomp2"
        new_q = np.zeros([3, 6])
        new_qh = np.zeros([3, 6])  #intermediate heights
        #	blockx, blocky, blockz, angle = self.get_color_block_world_coord('red')
        final_x = 0
        final_y = -220
        blockz = 19.25
        #	final_x = blockx
        #	final_y = blocky
        height = 40
        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 0) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[0] = self.roundoff(angles)
        new_qh[0] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 1) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[1] = self.roundoff(angles)
        new_qh[1] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 2) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[2] = self.roundoff(angles)
        new_qh[2] = self.roundoff(intermediate_angles)

        self.statemachine.setq_comp(new_q, new_qh)

    def generatecomp3(self):
        print "Entered generatecomp3"
        new_q = np.zeros([8, 6])
        new_qh = np.zeros([8, 6])  #intermediate heights
        #blockx, blocky, blockz, angle = self.get_color_block_world_coord('red')
        blockz = 19.5
        final_x = -150
        final_y = -89.5
        b = 43
        endCoord = [(final_x) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[0] = self.roundoff(angles)
        new_qh[0] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 1) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[1] = self.roundoff(angles)
        new_qh[1] = self.roundoff(intermediate_angles)
        print "Genereated FK: ", forwardKinematics(new_q[1][0], new_q[1][1],
                                                   new_q[1][2], new_q[1][3])

        endCoord = [(final_x + b * 2) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[2] = self.roundoff(angles)
        new_qh[2] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 3) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[3] = self.roundoff(angles)
        new_qh[3] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 4) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[4] = self.roundoff(angles)
        new_qh[4] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 5) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[5] = self.roundoff(angles)
        new_qh[5] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 6) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[6] = self.roundoff(angles)
        new_qh[6] = self.roundoff(intermediate_angles)

        endCoord = [(final_x + b * 7) / 10, (final_y) / 10, (blockz) / 10,
                    gripper_orientation]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[7] = self.roundoff(angles)
        new_qh[7] = self.roundoff(intermediate_angles)

        self.statemachine.setq_comp(new_q, new_qh)

    def generatecomp4(self):
        print "Entered generatecomp4"
        new_q = np.zeros([8, 6])
        new_qh = np.zeros([8, 6])  #intermediate heights
        blockx, blocky, blockz, angle = self.get_color_block_world_coord(
            'blue')
        final_x = 0
        final_y = -220
        height = 40
        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 0) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[0] = self.roundoff(angles)
        new_qh[0] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 1) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[1] = self.roundoff(angles)
        new_qh[1] = self.roundoff(intermediate_angles)
        print "Genereated FK: ", forwardKinematics(new_q[1][0], new_q[1][1],
                                                   new_q[1][2], new_q[1][3])

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 2) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[2] = self.roundoff(angles)
        new_qh[2] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 3) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

        new_q[3] = self.roundoff(angles)
        new_qh[3] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 4) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[4] = self.roundoff(angles)
        new_qh[4] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 5) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        new_q[5] = self.roundoff(angles)
        new_qh[5] = self.roundoff(intermediate_angles)

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 6) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        # hard code joints angles down below here
        #new_q[6] = self.roundoff(angles)
        #new_qh[6] = self.roundoff(intermediate_angles)
        new_q[6] = [1., 2., 66.5, 15.2, 0, 90.]
        new_qh[6] = [1., 15.8, 29., 34.5, 0, 90.]

        endCoord = [
            -(final_x) / 10, (final_y) / 10, (blockz + height * 7) / 10,
            gripper_orientation
        ]
        [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")
        #new_q[7] = self.roundoff(angles)
        #new_qh[7] = self.roundoff(intermediate_angles)
        new_q[7] = [1., 15.8, 29., 34.5, 0, 90.]
        new_qh[7] = [1., 23., 9., 34.5, 0, 90.]
        self.statemachine.setq_comp(new_q, new_qh)

    def generate_comp5(self):
        print "Entered generate_comp5"
        n = 3
        b = 6
        N = (n + 1) * n / 2
        coords = self.pyramid(n, b)
        new_q = np.zeros([N, 6])
        new_qh = np.zeros([N, 6])  #intermediate heights
        endCoord = np.zeros([N, 4])
        for i in range(0, N):
            endCoord[i] = [
                coords[0, i], coords[1, i], coords[2, i], gripper_orientation
            ]
            [new_q[i], new_qh[i]] = self.getIK(endCoord[i], 0, "placing")
        print "pyramid: ", coords
        self.statemachine.setq_comp(new_q, new_qh)

    def pyramid(self, nlayers, spacing):  #returns numpy array.
        coords = zeros((3, nlayers * int((nlayers + 1) / 2)))
        count = 0
        blockheight = 3.85
        for j in range(0, nlayers):
            for i in range(0, nlayers - j):
                print(nlayers - j - 1)
                coords[0][count] = 0 + spacing * (i - ((nlayers - j) / 2.0))
                coords[1][count] = -11
                coords[2][count] = (blockheight * j
                                    )  #define blockheight and offset
                count = count + 1
        return coords

    def competition(self):
        if (self.ui.btnUser11.text() == "Enter Competition Mode"):
            self.ui.btnUser11.setText("Enter Testing Mode")
            self.ui.btnUser7.setText("Draw")
            self.ui.btnUser1.setText("Competition 1")
            self.ui.btnUser2.setText("Competition 2")
            self.ui.btnUser3.setText("Competition 3")
            self.ui.btnUser4.setText("Competition 4")
            self.ui.btnUser5.setText("Competition 5")
            self.ui.btnUser2.setEnabled(True)
            self.ui.btnUser7.setEnabled(True)

        elif (self.ui.btnUser11.text() == "Enter Testing Mode"):
            self.ui.btnUser11.setText("Enter Competition Mode")
            self.ui.btnUser7.setText("Smooth Repeat")
            self.ui.btnUser1.setText("Configure Servos")
            self.ui.btnUser2.setText("Depth and RGB Calibration")
            self.ui.btnUser3.setText("Extrinsic Calibration")
            self.ui.btnUser4.setText("Block Detection")
            self.ui.btnUser5.setText("Repeat")
            self.ui.btnUser2.setEnabled(False)
            self.ui.btnUser7.setEnabled(False)

    def btn1(self):
        if (self.ui.btnUser1.text() == "Configure Servos"):
            self.rex.cfg_publish_default()
        elif (self.ui.btnUser1.text() == "Competition 1"):
            if (self.sanity_check(1)):
                self.getangles('red')
                self.statemachine.setmystatus(
                    "Competition 1", "picking",
                    "picking")  #mode="testing",action="picking")

    def btn2(self):
        if (self.ui.btnUser2.text() == "Depth and RGB Calibration"):
            self.deprgb_cali()
        elif (self.ui.btnUser2.text() == "Competition 2"):
            if (self.sanity_check(2)):
                self.generatecomp2()
                self.getangles('red')
                self.statemachine.setmystatus(
                    "Competition 2", "picking",
                    "picking")  #mode="testing",action="picking")

    def btn3(self):
        global declutter_index, declutter_competition
        if (self.ui.btnUser3.text() == "Extrinsic Calibration"):
            self.extrinsic_cali()
        elif (self.ui.btnUser3.text() == "Competition 3"):
            declutter_index = 0
            declutter_competition = "Competition 3"
            self.declutter()
            """
		if(self.sanity_check(3)):
			self.generatecomp3()
			self.getangles('black')		
			self.statemachine.setmystatus("Competition 3", "picking","picking")#mode="testing",action="picking")	
		"""

    def btn4(self):
        global declutter_index, declutter_competition
        if (self.ui.btnUser4.text() == "Block Detector"):
            self.video.blockDetector()
        elif (self.ui.btnUser4.text() == "Competition 4"):
            declutter_index = 0
            declutter_competition = "Competition 4"
            self.declutter()
            """
		if(self.sanity_check(4)):
			self.generatecomp4()
			self.getangles('black')		
			self.statemachine.setmystatus("Competition 4", "picking","picking")#mode="testing",action="picking")	
		"""

    def btn5(self):
        if (self.ui.btnUser5.text() == "Repeat"):
            self.repeat()
        elif (self.ui.btnUser5.text() == "Competition 5"):
            self.generate_comp5()
            self.getangles('all')
            self.statemachine.setmystatus(
                "Competition 5", "picking",
                "picking")  #mode="testing",action="picking")

    def declutter(self):
        blockx, blocky, blockz, angle = self.video.search_highest()
        levels_mm = np.float32(
            [0., 19.25, 57.75, 96.25, 134.75, 173.25, 211.75, 250.25])
        level = (blockz - 19.25) / 38.5
        print "level =  ", level, "and levels_mm = ", levels_mm

        if (declutter_index == 2):
            if (declutter_competition == "Competition 3"):
                if (self.sanity_check(3)):
                    self.generatecomp3()
                    self.getangles('black')
                    self.statemachine.setmystatus(
                        "Competition 3", "picking",
                        "picking")  #mode="testing",action="picking")
            elif (declutter_competition == "Competition 4"):
                if (self.sanity_check(4)):
                    self.generatecomp4()
                    self.getangles('black')
                    self.statemachine.setmystatus(
                        "Competition 4", "picking",
                        "picking")  #mode="testing",action="picking")

        if (level >= 1 and declutter_index < 4):
            endCoord = [(blockx) / 10, (blocky) / 10, (blockz) / 10,
                        gripper_orientation]
            print endCoord
            [angles, intermediate_angles] = self.getIK(endCoord, 0, "picking")
            print "from declutter IK output: ", [angles, intermediate_angles]
            angles = self.roundoff(angles)
            intermediate_angles = self.roundoff(intermediate_angles)
            if (not self.checkfound(angles)
                ):  # and not self.checkfound(intermediate_angles,True)):
                self.generatedeclutter()
                self.statemachine.setq(angles, intermediate_angles)
            self.statemachine.setmystatus(
                "Declutter", "picking",
                "picking")  #mode="testing",action="picking")
        else:
            if (declutter_competition == "Competition 3"):
                if (self.sanity_check(3)):
                    self.generatecomp3()
                    self.getangles('black')
                    self.statemachine.setmystatus(
                        "Competition 3", "picking",
                        "picking")  #mode="testing",action="picking")
            elif (declutter_competition == "Competition 4"):
                if (self.sanity_check(4)):
                    self.generatecomp4()
                    self.getangles('black')
                    self.statemachine.setmystatus(
                        "Competition 4", "picking",
                        "picking")  #mode="testing",action="picking")

    def generatedeclutter(self):
        print "Entered generatedeclutter"
        global declutter_index

        if (declutter_index < 4):
            x = [0, 250, 0, 180]
            y = [-240, -50, 240, -150]
            final_x = x[declutter_index]
            final_y = y[declutter_index]
            final_z = 19.25
            height = 20
            endCoord = [(final_x) / 10, (final_y) / 10,
                        (final_z + height) / 10, gripper_orientation]
            [angles, intermediate_angles] = self.getIK(endCoord, 0, "placing")

            new_q = self.roundoff(angles)
            new_qh = self.roundoff(intermediate_angles)
            declutter_index = declutter_index + 1
            self.statemachine.setq_comp(new_q, new_qh)

    def sanity_check(self, comp):
        success = False
        if (comp == 1 or comp == 2):  #search for red,blue,green
            colors = ["red", "blue", "green"]
            [pos_colors, success] = self.video.blockDetector(colors)
        elif (comp == 4 or comp == 3):  #search for red,blue,green
            colors = [
                "red", "blue", "green", "orange", "violet", "black", "yellow",
                "pink"
            ]
            [pos_colors, success] = self.video.blockDetector(colors)

        return success

    """main function"""
示例#3
0
class Gui(QtGui.QMainWindow):
    """ 
    Main GUI Class
    contains the main function and interfaces between 
    the GUI and functions
    """
    def __init__(self, parent=None):
        QtGui.QWidget.__init__(self, parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        """ Main Variables Using Other Classes"""
        self.rex = Rexarm()
        self.video = Video()
        self.statemachine = Statemachine()

        self.pos_idx_teach = 0
        """ Other Variables """
        self.last_click = np.float32([0, 0])
        self.check_move_time = 60
        self.check_time = 70
        self.check_move_counter = 0
        self.d_t = 10. * self.check_move_time / 1000.
        self.spe_coef = [0.0] * self.rex.num_joints
        self.pos_pre = [0.0] * self.rex.num_joints
        self.plan_point = 0
        self.move_to_point_cn = 0
        self.event3_color_check = [0] * 8
        self.event3_finish = 0
        self.event4_color_idx = 0
        self.vel = []
        self.event4_f_idx = 0
        self.event5_block_num = 0
        """ Set GUI to track mouse """
        QtGui.QWidget.setMouseTracking(self, True)
        """ 
        GUI timer 

        Creates a timer and calls update_gui() function 
        according to the given time delay (27ms) 
        """
        self._timer = QtCore.QTimer(self)
        self._timer.timeout.connect(self.update_gui)
        self._timer.start(27)
        """ 
        LCM Arm Feedback timer
        Creates a timer to call LCM handler for Rexarm feedback
        """
        self._timer2 = QtCore.QTimer(self)
        self._timer2.timeout.connect(self.rex.get_feedback)
        self._timer2.start()
        # self._timer_setplan = QtCore.QTimer(self)
        # self._timer_setplan.timeout.connect(self.setplan)
        # self._timer_teach = QtCore.QTimer(self)
        # self._timer_teach.timeout.connect(self.check_move)
        self._timer_waypoint = QtCore.QTimer(self)
        self._timer_waypoint.timeout.connect(self.move_to_point)
        self._timer_event1 = QtCore.QTimer(self)
        self._timer_event2 = QtCore.QTimer(self)
        self._timer_event3 = QtCore.QTimer(self)
        self._timer_event4 = QtCore.QTimer(self)
        self._timer_event5 = QtCore.QTimer(self)
        """ 
        Connect Sliders to Function
        TODO: CONNECT THE OTHER 5 SLIDERS IMPLEMENTED IN THE GUI 
        """
        self.ui.sldrBase.valueChanged.connect(self.sliderChange)
        self.ui.sldrShoulder.valueChanged.connect(self.sliderChange)
        self.ui.sldrElbow.valueChanged.connect(self.sliderChange)
        self.ui.sldrSpeed.valueChanged.connect(self.sliderChange)
        self.ui.sldrWrist.valueChanged.connect(self.sliderChange)
        self.ui.sldrGrip1.valueChanged.connect(self.sliderChange)
        self.ui.sldrGrip2.valueChanged.connect(self.sliderChange)
        self.ui.sldrMaxTorque.valueChanged.connect(self.sliderChange)
        """ Initial command of the arm to home position"""
        self.sliderChange()
        """ Connect Buttons to Functions 
        TODO: NAME AND CONNECT BUTTONS AS NEEDED
        """
        self.ui.btnUser1.setText("Configure Servos")
        self.ui.btnUser1.clicked.connect(self.rex.cfg_publish_default)
        self.ui.btnUser2.setText("Simple Calibration")
        self.ui.btnUser2.clicked.connect(self.simple_cal)
        # self.ui.btnUser3.setText("start teach and repeat")
        # self.ui.btnUser3.clicked.connect(self.teach_repeat_start)
        # self.ui.btnUser4.setText("record position")
        # self.ui.btnUser4.clicked.connect(self.teach_repeat_record)
        # self.ui.btnUser5.setText("play teach and repeat")
        # self.ui.btnUser5.clicked.connect(self.teach_repeat_play)
        # self.ui.btnUser6.setText("save rgb")
        # self.ui.btnUser6.clicked.connect(self.save_rgb)
        self.ui.btnUser3.setText("EVENT1")
        self.ui.btnUser3.clicked.connect(self.event1)
        self.ui.btnUser4.setText("EVENT2")
        self.ui.btnUser4.clicked.connect(self.event2)
        self.ui.btnUser5.setText("EVENT3")
        self.ui.btnUser5.clicked.connect(self.event3)
        self.ui.btnUser6.setText("EVENT4")
        self.ui.btnUser6.clicked.connect(self.event4)
        self.ui.btnUser7.setText("EVENT5")
        self.ui.btnUser7.clicked.connect(self.event5)
        self.ui.btnUser8.setText("depth test")
        self.ui.btnUser8.clicked.connect(self.video.blockDetector)
        self.ui.btnUser9.setText("recover")
        self.ui.btnUser9.clicked.connect(self.recover)
        self.ui.btnUser10.setText("click put")
        self.ui.btnUser10.clicked.connect(self.click_put)
        self.ui.btnUser12.setText("Emergency Stop!")
        self.ui.btnUser12.clicked.connect(self.estop)

    # def save_rgb(self):
    #     img = cv2.cvtColor(self.video.currentVideoFrame, cv2.COLOR_RGB2BGR)
    #     cv2.imwrite('real_example.png',img)
    # def teach_repeat_start(self):
    #     print "i enter"
    #     global Record_Value
    #     Record_Value = [[]]
    #     self.statemachine.idle(self.rex)
    #     # self.statemachine.moving(self.rex, self.rex.rexarm_IK([207,14, 30, np.pi/2],1) + [0.0, -10], [0.1,0.1,0.1,0.1,0.1,0.1])
    #     # print self.rex.rexarm_IK([-100,-100, 40, np.pi/2],1)

    # def teach_repeat_record(self):
    #     temp_record = [self.rex.joint_angles_fb[0],
    #     self.rex.joint_angles_fb[1],
    #     self.rex.joint_angles_fb[2],
    #     self.rex.joint_angles_fb[3],
    #     self.rex.joint_angles_fb[4]]
    #     if len(Record_Value[0]) == 0:
    #         Record_Value[0] = temp_record
    #     else :
    #         Record_Value.append(temp_record)

    # def teach_repeat_play(self):
    #     print Record_Value
    #     self._timer_teach.start(self.check_time)
    # # def check_move(self):
    #     print "111111"
    #     if len(Record_Value[0]) == 0:
    #         self.check_move_counter = 0
    #         self._timer_teach.stop()
    #         return
    #     if self.check_move_counter == 0:
    #         T = np.array([[1,0,0,0,0,0],[0,1,0,0,0,0],[0,0,2,0,0,0],[1,self.d_t,self.d_t**2,self.d_t**3,self.d_t**4,self.d_t**5],
    #             [0,1,2*self.d_t,3*self.d_t**2,4*self.d_t**3,5*self.d_t**4],[0,0,2,6*self.d_t,12*self.d_t**2,20*self.d_t**3]])
    #         for i in range(self.rex.num_joints):
    #             self.spe_coef[i] = np.dot(np.linalg.inv(T), np.array([[self.rex.joint_angles_fb[i]],[0.01],[0],
    #                 [Record_Value[self.pos_idx_teach][i]],[0.1],[0.]]))
    #         print "haha"
    #         print Record_Value[self.pos_idx_teach]
    #         print "fb"
    #         print self.rex.joint_angles_fb
    #         self.check_move_counter += 1
    #     pos = [0.] * self.rex.num_joints
    #     speed = [0.] * self.rex.num_joints
    #     for i in range(self.rex.num_joints):
    #         pos[i] = np.polynomial.polynomial.polyval((self.check_move_counter * self.check_move_time / 1000), self.spe_coef[i])
    #         if i == 3:
    #             speed[i] = abs(np.polynomial.polynomial.polyval((self.check_move_counter * self.check_move_time / 1000),
    #             np.polynomial.polynomial.polyder(self.spe_coef[i]))) / SPEED_AX12
    #         else:
    #             speed[i] = abs(np.polynomial.polynomial.polyval((self.check_move_counter * self.check_move_time / 1000),
    #             np.polynomial.polynomial.polyder(self.spe_coef[i]))) / SPEED_MX28 * 20

    #     if self.check_move_counter * self.check_move_time / 1000. <= self.d_t and np.linalg.norm(np.transpose(np.asarray(Record_Value[self.pos_idx_teach])) - np.transpose(np.asarray(self.rex.joint_angles_fb))) > 0.1:
    #         # and np.linalg.norm(Record_Value[self.pos_idx_teach] - np.asarray(self.rex.joint_angles_fb)) > 0.1
    #         if self.pos_pre != pos:
    #             self.statemachine.moving(self.rex, pos, speed)
    #         self.pos_pre = pos
    #         print "pos"
    #         print pos
    #         print "feedback"
    #         print self.rex.joint_angles_fb
    #         print "speed"
    #         print speed
    #         print self.check_move_counter
    #         # self.check_move_counter += 1
    #         if np.linalg.norm(np.transpose(np.asarray(pos)) - np.transpose(np.asarray(self.rex.joint_angles_fb)),axis = 1) < 0.01:
    #             self.check_move_counter += 1
    #     else:
    #         self.check_move_counter = 0
    #         if self.pos_idx_teach < len(Record_Value) - 1:
    #             self.pos_idx_teach += 1;
    #         else:
    #             print "i stop"
    #             self.pos_idx_teach = 0
    #             self._timer_teach.stop()
    #     print "222222"
    # def check_move(self):
    #     pos = 0
    #     if len(Record_Value[0]) == 0:
    #         self._timer_teach.stop()
    #         return
    #     print len(Record_Value[self.pos_idx_teach])
    #     print np.linalg.norm(np.asarray(Record_Value[self.pos_idx_teach]) - np.asarray(self.rex.joint_angles_fb[0:5]))
    #     if np.linalg.norm(np.transpose(np.asarray(Record_Value[self.pos_idx_teach])) - np.transpose(np.asarray(self.rex.joint_angles_fb[0:5]))) > 0.04:
    #         pos = Record_Value[self.pos_idx_teach] + [0]
    #         self.statemachine.moving(self.rex, pos,[0.1,0.1,0.1,0.1,0.1,0.1])
    #         print pos
    #     else:
    #         if self.pos_idx_teach < len(Record_Value) - 1:
    #             self.pos_idx_teach += 1;
    #         else:
    #             print "i stop"
    #             self.pos_idx_teach = 0
    #             self._timer_teach.stop()
    def recover(self):
        self.rex.moving_status = 0

    def move_to_point(self):
        pos = 0
        if len(self.rex.plan) == 0:
            self._timer_waypoint.stop()
            return
        # print len(self.rex.plan[self.pos_idx_teach])
        # print self.check_arrival(self.rex.plan[self.pos_idx_teach],self.rex.joint_angles_fb)
        if self.check_arrival(self.rex.plan[self.pos_idx_teach],
                              self.rex.joint_angles_fb
                              ) > 0.07 and self.move_to_point_cn < 40:
            pos = self.rex.plan[self.pos_idx_teach]
            vel = self.vel[self.pos_idx_teach]
            self.statemachine.moving(self.rex, pos, vel)
            self.move_to_point_cn += 1
            # print "load load"
            # print self.rex.load_fb
            # print "going to pos:"
            # print pos
        else:
            if self.move_to_point_cn >= 70:
                print "cant move to that point within %d s" % (
                    self.check_time * self.move_to_point_cn)
            self.move_to_point_cn = 0
            if self.pos_idx_teach < len(self.rex.plan) - 1:
                self.pos_idx_teach += 1
            else:
                print "i stop"
                self.pos_idx_teach = 0
                self.rex.plan = []
                self.rex.plan_status = 0
                self._timer_waypoint.stop()

    def click_grab(self):
        x = self.last_click[0]
        y = self.last_click[1]
        z = 0.
        world_z = 0.
        rw = np.array([[0], [0], [0]])
        M = self.video.aff_matrix
        M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
        v = np.array([[x], [y], [1]])
        DEP_xy = np.dot(M_RGBtoDEP, v)
        if ((DEP_xy[0] >= 0) and (DEP_xy[0] <= 640) and (DEP_xy[1] >= 0)
                and (DEP_xy[1] <= 480)):
            world_z = self.video.depth - 123.6 * np.tan(
                self.video.currentDepthFrame[int(DEP_xy[1])][int(DEP_xy[0])] /
                2842.5 + 1.1863)
            z = self.video.currentDepthFrame[int(DEP_xy[1])][int(DEP_xy[0])]
            rw = np.dot(
                np.linalg.inv(self.video.WORLDtoRGB[:, [0, 1, 3]]),
                np.subtract(v, self.video.WORLDtoRGB[:, 2:3] * (-world_z)))
        else:
            z = 0.0
        if rw[2] != 0:
            # self.statemachine.moving(self.rex, self.rex.rexarm_IK([rw[0]/rw[2],rw[1]/rw[2], world_z + 30, np.pi/2],1) + [0.0, -10], [0.1,0.1,0.1,0.1,0.1,0.1])
            t = self.rex.rexarm_IK(
                [rw[0] / rw[2], rw[1] / rw[2], world_z - 9, np.pi], 1)
            self.rex.plan = [[t[0], -200, -200, -200, 0, -50 * D2R],
                             [-200, -200, t[2], t[3], 0, -200],
                             [-200, t[1], -200, -200, 0, -200],
                             [-200, -200, -200, -200, 0, 7 * D2R],
                             [-200, 0, -200, -200, -200, -200],
                             [0, 0, 0, 0 * D2R, 0, -200]]
            self._timer_waypoint.start(self.check_time)
            print "plan"
            print self.rex.plan
        else:
            print "try again"

    def click_put(self):
        x = self.last_click[0]
        y = self.last_click[1]
        z = 0.
        world_z = 0.
        rw = np.array([[0], [0], [0]])
        M = self.video.aff_matrix
        M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
        v = np.array([[x], [y], [1]])
        DEP_xy = np.dot(M_RGBtoDEP, v)
        if ((DEP_xy[0] >= 0) and (DEP_xy[0] <= 640) and (DEP_xy[1] >= 0)
                and (DEP_xy[1] <= 480)):
            world_z = self.video.depth - 123.6 * np.tan(
                self.video.currentDepthFrame[int(DEP_xy[1])][int(DEP_xy[0])] /
                2842.5 + 1.1863)
            z = self.video.currentDepthFrame[int(DEP_xy[1])][int(DEP_xy[0])]
            rw = np.dot(
                np.linalg.inv(self.video.WORLDtoRGB[:, [0, 1, 3]]),
                np.subtract(v, self.video.WORLDtoRGB[:, 2:3] * (-world_z)))
        else:
            z = 0.0
        if rw[2] != 0:
            # self.statemachine.moving(self.rex, self.rex.rexarm_IK([rw[0]/rw[2],rw[1]/rw[2], world_z + 30, np.pi/2],1) + [0.0, -10], [0.1,0.1,0.1,0.1,0.1,0.1])
            t = self.rex.rexarm_IK(
                [rw[0] / rw[2], rw[1] / rw[2], world_z + 40, np.pi / 2], 1)
            self.rex.plan = [[t[0], -200, -200, -200, 0, -200],
                             [-200, -200, t[2], t[3], 0, -200],
                             [-200, t[1], -200, -200, 0, -200],
                             [-200, -200, -200, -200, 0, -20 * D2R],
                             [-200, 0, -200, -200, -200, -200],
                             [0, 0, 0, 0 * D2R, 0, -200]]
            self._timer_waypoint.start(self.check_time)
            print "plan"
            print self.rex.plan
        else:
            print "try again"

    def check_arrival(self, pos0, pos1):
        t = 0
        for i in range(len(pos0)):
            if pos0[i] != -200:
                if i != 5:
                    t = t + (pos0[i] - pos1[i])**2
                else:
                    t = t + 0.25 * (pos0[i] - pos1[i])**2
        return np.sqrt(t)

    def event1(self):
        print "entering event1"
        self.video.det_pts_ = []
        self.video.blockDetector()
        self.plan_point = 0
        self._timer_event1.timeout.connect(self.event1_timer)
        self._timer_event1.start(self.check_time)

    def event2(self):
        print "entering event2"
        self._timer_event2.stop()
        self.video.det_pts_ = []
        self.video.blockDetector()
        self.plan_point = 0
        self._timer_event2.timeout.connect(self.event2_timer)
        self._timer_event2.start(self.check_time)

    def event3(self):
        print "entering event3"
        # self.video.blockDetector()
        self.video.det_pts_ = []
        self.event3_finish = 0
        self.plan_point = 0
        self.event3_color_check = [0] * 8
        self._timer_event3.timeout.connect(self.event3_timer)
        self._timer_event3.start(self.check_time)

    def event4(self):
        print "entering event4"
        # self.video.blockDetector()
        self.video.det_pts_ = []
        self.plan_point = 0
        self.event4_color_idx = 0
        self.event4_f_idx = 0
        self._timer_event4.timeout.connect(self.event4_timer)
        self._timer_event4.start(self.check_time)

    def event5(self):
        print "entering event5"
        # self.video.blockDetector()
        self.video.det_pts_ = []
        self.plan_point = 0
        self.event5_block_num = 0
        self._timer_event5.timeout.connect(self.event5_timer)
        self._timer_event5.start(self.check_time)

    def event1_timer(self):
        if self.rex.moving_status == 1:
            self._timer_event1.stop()
            self.rex.plan = []
            self.vel = []
            self._timer_waypoint.stop()

        if len(self.video.det_pts_) == 0:
            print "no block detected"
            self.plan_point = 0
            self._timer_event1.stop()
        else:
            if self.rex.plan_status == 0 and self.plan_point < len(
                    self.video.det_pts_):
                x = self.video.det_pts_[self.plan_point].rgb_real_pos[0]
                y = self.video.det_pts_[self.plan_point].rgb_real_pos[1]
                print "grapping %d %d %d" % (self.plan_point, x, y)
                world_z = self.video.det_pts_[self.plan_point].depth
                rw = np.array([[0], [0], [0]])
                M = self.video.aff_matrix
                M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
                v = np.array([[x], [y], [1]])
                t = self.rex.rexarm_IK([x, y, world_z, np.pi / 2], 1)
                t1 = self.rex.rexarm_IK([x, -y, world_z, np.pi / 2], 1)
                # if t[0] < 0:
                #     t[0] = np.pi + t[0]
                #     t[1] =  0 - t[1]
                #     t[2] = 0 - t[2]
                #     t[3] = 0 - t[3]
                # if t1[0] < 0:
                #     t1[0] = np.pi + t1[0]
                #     t1[1] = 0 - t1[1]
                #     t1[2] = 0 - t1[2]
                #     t1[3] = 0 - t1[3]
                # self.rex.plan = [[t[0],-200,-200,-200,0,-30*D2R],[-200,-200,t[2],t[3],0,-200],[-200,t[1],-200,-200,0,-200],[-200,-200,-200,-200,0,10*D2R],[-200,0,-200,-200,-200,-200],[-200,0,0,0*D2R,0,-200]]
                self.rex.plan = [[t[0], -200, t[2], t[3], 0, -30 * D2R],
                                 [-200, t[1], -200, -200, 0, -200],
                                 [-200, -200, -200, -200, 0, 8 * D2R],
                                 [
                                     -200, t1[1] -
                                     55 * D2R if t1[1] - 55 * D2R > 0 else 0,
                                     -200, -200, -200, -200
                                 ]]
                # t = self.rex.rexarm_IK([x,-y, world_z, np.pi/2],1)
                self.rex.plan.extend(
                    [[t1[0], -200, -200, -200, 0, -200],
                     [-200, -200, t1[2], t1[3], 0, -200],
                     [-200, t1[1], -200, -200, 0, -200],
                     [-200, -200, -200, -200, 0, -15 * D2R],
                     [
                         -200, t1[1] - 55 * D2R if t1[1] - 55 * D2R > 0 else 0,
                         t1[2] + 5 * D2R, -200, -200, -200
                     ]])
                self.vel = [[0.3, 0.25, 0.15, 0.15, 0.7, 0.8]] * len(
                    self.rex.plan)
                self.plan_point += 1
                self._timer_waypoint.start(self.check_time)
                self.rex.plan_status = 1
            elif self.plan_point == len(
                    self.video.det_pts_) and self.rex.plan_status == 0:
                print "event1 stops"
                self._timer_event1.stop()
                self.rex.plan = []
                self.vel = []
            else:
                pass

    def event2_timer(self):
        if self.rex.moving_status == 1:
            self._timer_event2.stop()
            self.rex.plan = []
            self._timer_waypoint.stop()
        if len(self.video.det_pts_) == 0:
            print "no block detected"
            self.plan_point = 0
            self._timer_event2.stop()
        else:
            if self.rex.plan_status == 0 and self.plan_point < len(
                    self.video.det_pts_):
                x = self.video.det_pts_[self.plan_point].rgb_real_pos[0]
                y = self.video.det_pts_[self.plan_point].rgb_real_pos[1]
                print "grapping %d %d %d" % (self.plan_point, x, y)
                world_z = self.video.det_pts_[self.plan_point].depth
                rw = np.array([[0], [0], [0]])
                M = self.video.aff_matrix
                M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
                v = np.array([[x], [y], [1]])
                t = self.rex.rexarm_IK([x, y, world_z, np.pi / 2], 1)
                t1 = self.rex.rexarm_IK(
                    [EVEN2_X, EVEN2_Y, 25 + 38 * self.plan_point, np.pi / 2],
                    1)
                self.rex.plan = [[t[0], -200, t[2], t[3], 0, -20 * D2R],
                                 [-200, t[1], -200, -200, 0, -200],
                                 [-200, -200, -200, -200, 0, 8 * D2R],
                                 [
                                     -200, t1[1] -
                                     55 * D2R if t1[1] - 55 * D2R > 0 else 0,
                                     t1[2], t1[3], -200, -200
                                 ]]
                self.rex.plan.extend(
                    [[t1[0], -200, t1[2], t1[3], 0, -200],
                     [-200, t1[1], -200, -200, 0, -200],
                     [-200, -200, -200, -200, 0, -20 * D2R],
                     [
                         -200, t1[1] - 55 * D2R if t1[1] - 55 * D2R > 0 else 0,
                         t1[2] + 5 * D2R, -200, -200, -200
                     ]])
                self.vel = [[0.3, 0.25, 0.15, 0.15, 0.7, 0.8]] * len(
                    self.rex.plan)
                self.plan_point += 1
                self._timer_waypoint.start(self.check_time)
                self.rex.plan_status = 1
            elif self.plan_point == len(
                    self.video.det_pts_) and self.rex.plan_status == 0:
                print "event2 stops"
                self._timer_event2.stop()
                self.rex.plan = []
                self.vel = []
            else:
                pass

    def event3_timer(self):
        if self.rex.moving_status == 1:
            self._timer_event3.stop()
            self.rex.plan = []
            self._timer_waypoint.stop()
        if len(self.video.det_pts_) == 0:
            print "begin to detect"
            self.video.blockDetector()
            points_temp = []
            for points_i in self.video.det_pts_:
                if self.event3_color_check[
                        points_i.color] == 0 and points_i.rgb_real_pos[0] < 0:
                    self.event3_color_check[points_i.color] = 1
                    points_temp.append(points_i)
            self.video.det_pts_ = points_temp
            self.plan_point = 0
            if len(self.video.det_pts_) == 0:
                if self.event3_finish == 0 and self.rex.plan_status == 0:
                    print "pushing blocks"
                    self.rex.plan = [[
                        -115 * D2R, 68 * D2R, 19 * D2R, 76 * D2R, -115 * D2R, 0
                    ], [
                        -122 * D2R, 57 * D2R, 47 * D2R, 76 * D2R, -122 * D2R, 0
                    ]]
                    self.vel = [[0.2, 0.1, 0.05, 0.05, 0.3, 0.3]] * len(
                        self.rex.plan)
                    self._timer_waypoint.start(self.check_time)
                    self.rex.plan_status = 1
                    self.event3_finish = 1
                elif self.event3_finish == 1 and self.rex.plan_status == 0:
                    print "event finishes"
                    self._timer_event3.stop()
                    self.rex.plan = []
                    self.vel = []
                return
            # self._timer_event.stop()

        if self.rex.plan_status == 0 and self.plan_point < len(
                self.video.det_pts_):
            x = self.video.det_pts_[self.plan_point].rgb_real_pos[0]
            y = self.video.det_pts_[self.plan_point].rgb_real_pos[1]
            print "grapping %d %d %d" % (self.plan_point, x, y)
            world_z = self.video.det_pts_[self.plan_point].depth
            rw = np.array([[0], [0], [0]])
            M = self.video.aff_matrix
            M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
            v = np.array([[x], [y], [1]])
            if np.arctan2(y, x) < -np.pi / 2 and np.arctan2(
                    y, x) > -3 * np.pi / 4:
                t = self.rex.rexarm_IK([x, y, world_z, np.pi / 2], 1)
            else:
                t = self.rex.rexarm_IK([x, y, world_z - 5, np.pi / 2], 1)
            self.rex.plan = [[t[0] - 2 * D2R, -200, -200, -200, 0, -30 * D2R],
                             [-200, -200, t[2], t[3], 0, -200],
                             [-200, t[1], -200, -200, 0, -200],
                             [-200, -200, -200, -200, 0, 8 * D2R],
                             [-200, 0, -200, -200, -200, -200],
                             [-200, 0, 0, 0 * D2R, 0, -200]]
            self.vel = [[0.2, 0.15, 0.13, 0.13, 0.7, 0.7]] * len(self.rex.plan)
            color_idx = self.video.det_pts_[self.plan_point].color
            t = self.rex.rexarm_IK(
                [EVENT3_X, EVENT3_Y[color_idx], 24 + 38, np.pi / 2], 1)
            t1 = self.rex.rexarm_IK(
                [EVENT3_X, EVENT3_Y[color_idx], 24, np.pi / 2], 1)
            # self.rex.plan.extend([[t[0],-200,-200,-200,t[0],-200],[-200,-200,t[2],t[3],-200,-200],[-200,t[1],-200,-200,-200,-200],[-200,-200,-200,-200,-200,-50*D2R],[-200,0,-200,-200,-200,-200],[-200,0,0,0*D2R,0,-200]])
            t4 = t[0] + np.pi / 2 if t[0] < 0 else t[0] - 3 * np.pi / 2
            self.rex.plan.extend([[t[0], -200, -200, -200, t4, -200],
                                  [-200, -200, t[2], t[3], -200, -200],
                                  [-200, t[1], -200, -200, -200, -200],
                                  [t1[0], t1[1], t1[2], t1[3], t4, -200],
                                  [-200, -200, -200, -200, -200, -50 * D2R],
                                  [-200, 0, -200, -200, -200, -200],
                                  [-200, 0, 0, 0 * D2R, 0, -200]])
            self.vel.extend([[0.2, 0.15, 0.13, 0.13, 0.7, 0.7],
                             [0.2, 0.15, 0.13, 0.13, 0.7, 0.7],
                             [0.2, 0.15, 0.13, 0.13, 0.7, 0.7],
                             [0.07, 0.05, 0.05, 0.05, 0.7, 0.5],
                             [0.2, 0.15, 0.13, 0.13, 0.7, 0.7],
                             [0.2, 0.15, 0.13, 0.13, 0.7, 0.7],
                             [0.2, 0.15, 0.13, 0.13, 0.7, 0.7]])
            self.event3_color_check[self.video.det_pts_[
                self.plan_point].color] = 1
            self.plan_point += 1
            self._timer_waypoint.start(self.check_time)
            self.rex.plan_status = 1
        elif self.plan_point == len(
                self.video.det_pts_) and self.rex.plan_status == 0:
            print "event3 finish once"
            # self._timer_event.stop()
            self.rex.plan = []
            self.vel = []
            self.plan_point = 0
            self.video.det_pts_ = []
        else:
            pass

    def event4_timer(self):
        color_status = 0
        if self.rex.moving_status == 1:
            self._timer_event4.stop()
            self.rex.plan = []
            self._timer_waypoint.stop()
        if len(self.video.det_pts_) == 0:
            print "begin to detect"
            self.video.blockDetector()
            points_temp = []
            if self.event4_color_idx == 8:
                print "event finishes"
                self._timer_event4.stop()
                self.rex.plan = []
                self.vel = []
                return
            for points_i in self.video.det_pts_:
                if points_i.color == RE_COLOR[self.event4_color_idx]:
                    points_temp.append(points_i)
                    self.event4_color_idx += 1
                    color_status = 1
                    break
            if color_status == 0:
                for points_i in self.video.det_pts_:
                    if points_i.depth > 50 and points_i.rgb_real_pos[0] < 0:
                        points_temp.append(points_i)
                        break
            self.video.det_pts_ = points_temp
            self.plan_point = 0
            # self._timer_event.stop()

        if self.rex.plan_status == 0 and self.plan_point < len(
                self.video.det_pts_):
            x = self.video.det_pts_[self.plan_point].rgb_real_pos[0]
            y = self.video.det_pts_[self.plan_point].rgb_real_pos[1]
            print "grapping %d %d %d" % (self.plan_point, x, y)
            world_z = self.video.det_pts_[self.plan_point].depth
            rw = np.array([[0], [0], [0]])
            M = self.video.aff_matrix
            M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
            v = np.array([[x], [y], [1]])
            t = self.rex.rexarm_IK([x, y, world_z, np.pi / 2], 1)

            self.rex.plan = [[t[0], -200, -200, -200, 0, -30 * D2R],
                             [-200, -200, t[2], t[3], 0, -200],
                             [-200, t[1], -200, -200, 0, -200],
                             [-200, -200, -200, -200, 0, 8 * D2R],
                             [-200, 0, -200, -200, -200, -200],
                             [-200, 0, 0, 0 * D2R, 0, -200]]
            self.vel = [[0.25, 0.2, 0.18, 0.18, 0.7, 0.7]] * len(self.rex.plan)
            color_idx = self.video.det_pts_[self.plan_point].color
            if color_status == 1:
                t = self.rex.rexarm_IK([
                    EVENT4_X, EVENT4_Y,
                    (self.event4_color_idx - 1) * 38 + 24, np.pi / 2
                ], 1)
            else:
                t = self.rex.rexarm_IK([
                    EVENT4_XF[self.event4_f_idx], EVENT4_YF[self.event4_f_idx],
                    24, np.pi / 2
                ], 1)
                self.event4_f_idx += 1
            self.rex.plan.extend([[t[0], -200, -200, -200, 0, -200],
                                  [-200, -200, t[2], t[3], 0, -200],
                                  [-200, t[1], -200, -200, 0, -200],
                                  [-200, -200, -200, -200, 0, -20 * D2R],
                                  [-200, 0, t[2], -200, -200, -200],
                                  [-200, 0, 0, 0 * D2R, 0, -200]])
            self.vel.extend([[0.25, 0.2, 0.18, 0.18, 0.7, 0.7]] * 6)
            self.plan_point += 1
            self._timer_waypoint.start(self.check_time)
            self.rex.plan_status = 1
        elif self.plan_point == len(
                self.video.det_pts_) and self.rex.plan_status == 0:
            print "event4 finish once"
            # self._timer_event.stop()
            self.rex.plan = []
            self.vel = []
            self.plan_point = 0
            self.video.det_pts_ = []
        else:
            pass

    def event5_timer(self):
        if self.rex.moving_status == 1:
            self._timer_event5.stop()
            self.rex.plan = []
            self._timer_waypoint.stop()

        if len(self.video.det_pts_) == 0:
            if self.event5_block_num == 29:
                print "event finishes"
                self._timer_event5.stop()
                self.rex.plan = []
                self.vel = []
                return
            print "begin to detect"
            self.video.blockDetector()
            points_temp = []
            points_max = 0
            for points_i in self.video.det_pts_:
                if points_i.rgb_real_pos[0] > 0 and abs(
                        np.arctan2(points_i.rgb_real_pos[1],
                                   points_i.rgb_real_pos[0])) > points_max:
                    points_temp.append(points_i)
                    points_max = abs(
                        np.arctan2(points_i.rgb_real_pos[1],
                                   points_i.rgb_real_pos[0]))
            # print points_i.depth
            self.video.det_pts_ = [points_temp[-1]]
            self.plan_point = 0

        if self.rex.plan_status == 0 and self.plan_point < len(
                self.video.det_pts_):
            x = self.video.det_pts_[self.plan_point].rgb_real_pos[0]
            y = self.video.det_pts_[self.plan_point].rgb_real_pos[1]
            print "grapping %d %d %d" % (self.plan_point, x, y)
            world_z = self.video.det_pts_[self.plan_point].depth
            rw = np.array([[0], [0], [0]])
            M = self.video.aff_matrix
            M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
            v = np.array([[x], [y], [1]])
            t = self.rex.rexarm_IK([x, y, world_z, np.pi / 2], 1)
            if x > 0 and y > 0:
                t[0] -= 2 * D2R
            self.rex.plan = [[t[0], -200, -200, -200, 0, -30 * D2R],
                             [-200, -200, t[2], t[3], 0, -200],
                             [-200, t[1], -200, -200, 0, -200],
                             [-200, -200, -200, -200, 0, 8 * D2R],
                             [-200, 0, -200, -200, -200, -200],
                             [-200, 0, 0, 0 * D2R, 0, -200]]
            self.vel = [[0.3, 0.2, 0.18, 0.18, 0.7, 0.7]] * len(self.rex.plan)
            color_idx = self.video.det_pts_[self.plan_point].color
            if self.event5_block_num < 28:
                t = self.rex.rexarm_IK([
                    EVENT5_X[self.event5_block_num],
                    EVENT5_Y[self.event5_block_num],
                    29 + EVENT5_Z[self.event5_block_num] * 38, np.pi / 2
                ], 1)
            else:
                t = self.rex.rexarm_IK([
                    EVENT5_X[27], EVENT5_Y[27], 29 + EVENT5_Z[27] * 38 + 38,
                    np.pi / 2
                ], 1)
            self.rex.plan.extend([[t[0], -200, -200, -200, 0, -200],
                                  [-200, -200, t[2], t[3], 0, -200],
                                  [-200, t[1], -200, -200, 0, -200],
                                  [-200, -200, -200, -200, 0, -20 * D2R],
                                  [-200, 0, t[2] + 5 * D2R, -200, -200, -200],
                                  [-200, 0, 0, 0 * D2R, 0, -200]])
            self.vel.extend([[0.285, 0.2, 0.18, 0.18, 0.7, 0.7]] * 6)
            self.plan_point += 1
            self.event5_block_num += 1
            self._timer_waypoint.start(self.check_time)
            self.rex.plan_status = 1
        elif self.plan_point == len(
                self.video.det_pts_) and self.rex.plan_status == 0:
            print "event5 finish once"
            # self._timer_event.stop()
            self.rex.plan = []
            self.vel = []
            self.plan_point = 0
            self.video.det_pts_ = []
        else:
            pass

    def estop(self):
        self.statemachine.estop(self.rex)

    def update_gui(self):
        """ 
        update_gui function
        Continuously called by timer1 
        """
        """ Renders the video frames
            Displays a dummy image if no video available
            HINT: you can use the radio buttons to select different
            video sources like is done below
        """

        if (self.video.kinectConnected == 1):
            try:
                self.video.captureVideoFrame()
                self.video.captureDepthFrame()
            except TypeError:
                self.video.kinectConnected = 0
                self.video.loadVideoFrame()
                self.video.loadDepthFrame()

        if (self.ui.radioVideo.isChecked()):
            self.ui.videoFrame.setPixmap(self.video.convertFrame())
            # x = QtGui.QWidget.mapFromGlobal(self,QtGui.QCursor.pos()).x()
            # y = QtGui.QWidget.mapFromGlobal(self,QtGui.QCursor.pos()).y()
            # if ((x > MIN_X) and (x < MAX_X) and (y > MIN_Y) and (y < MAX_Y)):
            #     img = self.video.currentVideoFrame
            #     hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
            #     y -= 30
            #     x -= 310
            #     h = hsv[y][x][0]
            #     s = hsv[y][x][1]
            #     v = hsv[y][x][2]
            #     global hp, xp, yp
            #     if ((xp - x)*(xp - x) > 4 and (yp - y)*(yp - y) > 4) or (hp - h)*(hp - h) >= 1:
            #         print "x: %d y: %d hsv : (%d, %d, %d)" % (x,y,h,s,v)
            #     xp = x
            #     yp = y
            #     hp = h
        if (self.ui.radioDepth.isChecked()):
            self.ui.videoFrame.setPixmap(self.video.convertDepthFrame())
        """ 
        Update GUI Joint Coordinates Labels
        """
        self.rex.rexarm_FK()
        self.ui.rdoutBaseJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[0] * R2D)))
        self.ui.rdoutShoulderJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[1] * R2D)))
        self.ui.rdoutElbowJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[2] * R2D)))
        self.ui.rdoutWristJC.setText(
            str("%.2f" % (self.rex.joint_angles_fb[3] * R2D)))
        self.ui.rdoutX.setText(str("%.2f" % (self.rex.endeffector_global[0])))
        self.ui.rdoutY.setText(str("%.2f" % (self.rex.endeffector_global[1])))
        self.ui.rdoutZ.setText(str("%.2f" % (self.rex.endeffector_global[2])))
        self.ui.rdoutT.setText(str("%.2f" % (self.rex.endeffector_global[3])))
        """ 
        Mouse position presentation in GUI
        TODO: after implementing workspace calibration 
        display the world coordinates the mouse points to 
        in the RGB video image.
        """
        x = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).x()
        y = QtGui.QWidget.mapFromGlobal(self, QtGui.QCursor.pos()).y()
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)):
            self.ui.rdoutMousePixels.setText("(-,-,-)")
            self.ui.rdoutMouseWorld.setText("(-,-,-)")
        else:
            x = x - MIN_X
            y = y - MIN_Y
            z = 0.
            world_z = 0.
            rw = np.array([[0], [0], [0]])
            self.ui.rdoutMousePixels.setText("(%.0f,%.0f,%f)" % (x, y, z))
            if (self.video.cal_flag == 2):
                M = self.video.aff_matrix
                M_RGBtoDEP = self.video.aff_matrix_RGBtoDEP
                v = np.array([[x], [y], [1]])
                DEP_xy = np.dot(M_RGBtoDEP, v)
                if ((DEP_xy[0] >= 0) and (DEP_xy[0] <= 640)
                        and (DEP_xy[1] >= 0) and (DEP_xy[1] <= 480)):
                    world_z = self.video.depth - 123.6 * np.tan(
                        self.video.currentDepthFrame[int(DEP_xy[1])][int(
                            DEP_xy[0])] / 2842.5 + 1.1863)
                    z = self.video.currentDepthFrame[int(DEP_xy[1])][int(
                        DEP_xy[0])]
                    rw = np.dot(
                        np.linalg.inv(self.video.WORLDtoRGB[:, [0, 1, 3]]),
                        np.subtract(v, self.video.WORLDtoRGB[:, 2:3] *
                                    (-world_z)))
                else:
                    z = 0.0
                if rw[2] != 0:
                    self.ui.rdoutMouseWorld.setText(
                        "(%.0f,%.0f, %.0f)" %
                        (rw[0] / rw[2], rw[1] / rw[2], world_z))
                else:
                    self.ui.rdoutMouseWorld.setText("(-,-,-)")
            else:
                self.ui.rdoutMouseWorld.setText("(-,-,-)")
        """ 
        Updates status label when rexarm playback is been executed.
        This can be extended to include other appropriate messages
        """
        if (self.rex.plan_status == 1):
            self.ui.rdoutStatus.setText("Playing Back - Waypoint %d" %
                                        (self.rex.wpt_number + 1))

    def sliderChange(self):
        """ 
        Function to change the slider labels when sliders are moved
        and to command the arm to the given position
        """
        self.ui.rdoutBase.setText(str(self.ui.sldrBase.value()))
        self.ui.rdoutShoulder.setText(str(self.ui.sldrShoulder.value()))
        self.ui.rdoutElbow.setText(str(self.ui.sldrElbow.value()))
        self.ui.rdoutWrist.setText(str(self.ui.sldrWrist.value()))
        # self.ui.rdoutGrip1.setText(str(self.ui.sldrGrip1.value()))
        # self.ui.rdoutGrip2.setText(str(self.ui.sldrGrip2.value()))

        self.ui.rdoutTorq.setText(str(self.ui.sldrMaxTorque.value()) + "%")
        self.ui.rdoutSpeed.setText(str(self.ui.sldrSpeed.value()) + "%")
        self.rex.torque_multiplier = self.ui.sldrMaxTorque.value() / 100.0
        self.rex.speed_multiplier = self.ui.sldrSpeed.value() / 100.0
        self.rex.joint_angles[0] = self.ui.sldrBase.value() * D2R
        self.rex.joint_angles[1] = self.ui.sldrShoulder.value() * D2R
        self.rex.joint_angles[2] = self.ui.sldrElbow.value() * D2R
        self.rex.joint_angles[3] = self.ui.sldrWrist.value() * D2R
        self.rex.joint_angles[4] = self.ui.sldrGrip1.value() * D2R
        self.rex.joint_angles[5] = self.ui.sldrGrip2.value() * D2R
        # self.rex.joint_angles[6] = self.ui.sldrGrip2.value()*D2R
        self.rex.cmd_publish()

    def mousePressEvent(self, QMouseEvent):
        """ 
        Function used to record mouse click positions for calibration 
        """
        """ Get mouse posiiton """
        x = QMouseEvent.x()
        y = QMouseEvent.y()
        """ If mouse position is not over the camera image ignore """
        if ((x < MIN_X) or (x > MAX_X) or (y < MIN_Y) or (y > MAX_Y)): return
        """ Change coordinates to image axis """
        self.last_click[0] = x - MIN_X
        self.last_click[1] = y - MIN_Y
        """ If calibration has been performed """
        if (self.video.cal_flag == 1):
            """ Save last mouse coordinate """
            self.video.mouse_coord[self.video.mouse_click_id] = [(x - MIN_X),
                                                                 (y - MIN_Y)]
            """ Update the number of used poitns for calibration """
            self.video.mouse_click_id += 1
            """ Update status label text """
            self.ui.rdoutStatus.setText("Affine Calibration: Click Point %d" %
                                        (self.video.mouse_click_id + 1))
            """ 
            If the number of click is equal to the expected number of points
            computes the affine calibration.
            
            LAB TASK: Change this code to use your workspace calibration routine
            and NOT the simple calibration function as is done now.
            """
            if (self.video.mouse_click_id == 2 * self.video.cal_points):
                """ 
                Update status of calibration flag and number of mouse
                clicks
                """
                self.video.cal_flag = 2
                self.video.mouse_click_id = 0
                """ Perform affine calibration with OpenCV """
                self.video.aff_matrix = cv2.getAffineTransform(
                    self.video.mouse_coord[0:3], self.video.real_coord[0:3])
                self.video.aff_matrix_RGBtoDEP = cv2.getAffineTransform(
                    self.video.mouse_coord[0:3], self.video.mouse_coord[5:8])
                self.video.aff_matrix_DEPtoRGB = cv2.getAffineTransform(
                    self.video.mouse_coord[5:8], self.video.mouse_coord[0:3])
                object_points = np.insert(self.video.real_coord,
                                          2, [0, -38, -76, -76, -38],
                                          axis=1)
                image_points = np.array(
                    self.video.mouse_coord[0:self.video.cal_points])
                reval, rvec, tvec = cv2.solvePnP(object_points, image_points,
                                                 self.video.intrinsic,
                                                 self.video.distortion,
                                                 cv2.SOLVEPNP_EPNP)
                self.video.WORLDtoRGB = np.dot(
                    self.video.intrinsic,
                    np.insert(cv2.Rodrigues(rvec)[0], [3], tvec, axis=1))
                t = np.dot(self.video.WORLDtoRGB,
                           np.array([[-100.0], [100.0], [-114.], [1.]]))
                self.video.WORLDtoRGB /= t[2]
                # print t[2]
                z1_temp = np.tan(self.video.currentDepthFrame[int(
                    self.video.mouse_coord[5][1])][int(
                        self.video.mouse_coord[5][0])] / 2842.5 + 1.1863)
                z2_temp = np.tan(self.video.currentDepthFrame[int(
                    self.video.mouse_coord[6][1])][int(
                        self.video.mouse_coord[6][0])] / 2842.5 + 1.1863)
                z3_temp = np.tan(self.video.currentDepthFrame[int(
                    self.video.mouse_coord[7][1])][int(
                        self.video.mouse_coord[7][0])] / 2842.5 + 1.1863)
                z4_temp = np.tan(self.video.currentDepthFrame[int(
                    self.video.mouse_coord[8][1])][int(
                        self.video.mouse_coord[8][0])] / 2842.5 + 1.1863)
                z5_temp = np.tan(self.video.currentDepthFrame[int(
                    self.video.mouse_coord[9][1])][int(
                        self.video.mouse_coord[9][0])] / 2842.5 + 1.1863)
                # print z1_temp
                # print z2_temp
                # print z3_temp
                # print z4_temp
                # self.video.depth = 123.6 * (z1_temp + z2_temp) / 2
                self.video.depth = 123.6 * (z1_temp + z2_temp + z3_temp +
                                            z4_temp + z5_temp) / 5 + 45.5
                """ Updates Status Label to inform calibration is done """
                self.ui.rdoutStatus.setText("Waiting for input")

    def simple_cal(self):
        """ 
        Function called when affine calibration button is called.
        Note it only chnages the flag to record the next mouse clicks
        and updates the status text label 
        """
        self.video.cal_flag = 1
        self.ui.rdoutStatus.setText("Affine Calibration: Click Point %d" %
                                    (self.video.mouse_click_id + 1))