Esempio n. 1
0
    fnames = [f for f in fnames if f.find('image_') >= 0]
    n = len(fnames)/2

    # Store some values in order to keep track of FPS
    if (showFPS):
        startTime = time()
        FPS = 0
        lastI = 0

    # Get our plot points ready
    timePoints = [[], []]
    plotPoints = [[[], [], []], [[], [], []]]

    # Create the mask and table model
    mask = ~np.bool8(cv2.imread(os.path.join(folder, 'mask.png'), -1))
    tablemodel = util.buildMinMap(os.path.join(folder, 'table'))

    i = 0
    waitAmount = 5
    
    handList = None
    camShifter = None
    colors = None

    # Loop until we are out of images
    while (i < n):
        
        print "Processing Frame ", i

        # Show the FPS if desired
        if (showFPS):
Esempio n. 2
0
def getHands(image, depth, camShifter = None, colors = None, mask = None, tablemodel = None, showImages = False):

    # Make sure the mask and table model are available.  Otherwise, try to create them
    if (mask is None or tablemodel is None):
        mask = ~np.bool8(cv2.imread(os.path.join(folder, 'mask.png'), -1))
        tablemodel = util.buildMinMap(os.path.join(folder, 'table'))

    # If there is not camShifter currently running, we create a new one and start from scratch
    if (camShifter is None):
        camShifter = []
        colors = []
        for i in range(NUM_HANDS):
            camShifter.append(CamShiftTracker())
            colors.append((random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)))

    # Same for colors... choose random colors if there are not passed in
    if (colors is None):
        colors = []
        for i in range(NUM_HANDS):
            colors.append((random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)))

    # Make copies of the original image
    originalImage = image.copy()
    original = image.copy()
    shp = (image.shape[0], image.shape[1])

    # Split into HSV
    hue = np.zeros(shp, dtype='uint8')
    sat = np.zeros(shp, dtype='uint8')
    val = np.zeros(shp, dtype='uint8')
    hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    cv2.split(hsv, (hue, sat, val))

    hands = np.zeros(shp, dtype='uint8')

    # If we have debug enabled, show the images
    if DEBUG:
        cv2.imshow('Hue', hue)
        cv2.imshow('Saturation', sat)
        cv2.imshow('Value', val)

    # Find the contours of the depth segmentation that intersect the top of the screen
    # This is a bit of a slow process unfortunately
    newDep = cv2.convertScaleAbs(cv2.absdiff(depth, tablemodel))
    ret, depThresh = cv2.threshold(newDep, 12, 255, cv2.THRESH_BINARY)
    depConts, depHier = cv2.findContours(depThresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    depRects = []
    for cont in depConts:
        rect = cv2.boundingRect(cont)
        newRect = (rect[1], rect[0], rect[0] + rect[2], rect[1] + rect[3])
        if (area((rect[0], rect[1], rect[0] + rect[2], rect[1] + rect[3])) > 2000):
            depRects.append(newRect)
    depRects.sort()

    im = np.zeros(shp + (3,), dtype = "uint8")
    for j in range(NUM_HANDS):
        if (len(depRects) > j and depRects[j][0] < 20):
            depHand = (depRects[j][1], 0, depRects[j][2], depRects[j][3])
            cv2.rectangle(image, (depHand[0], depHand[1]), (depHand[2], depHand[3]), (255, 0, 0))

            im[depHand[1] : depHand[3], depHand[0] : depHand[2], :] = 1

    # Multiply the image by the binary depth segment
    outputOld = cv2.multiply(image, im)
    ret, output = cv2.threshold(outputOld, 0, 255, cv2.THRESH_BINARY)
    output1 = np.zeros(shp, dtype='uint8')
    output2 = np.zeros(shp, dtype='uint8')
    output3 = np.zeros(shp, dtype='uint8')
    cv2.split(output, (output1, output2, output3))

    meanVal = int(round(cv2.mean(val)[0])) - 30

    # Segment by HSV within acceptable ranges of human skin color
    sat = cv2.inRange(sat, np.array((68)), np.array((200)))
    hue = cv2.inRange(hue, np.array((0)), np.array((22)))
    val = cv2.inRange(val, np.array((meanVal)), np.array((255)))

    # Multiply all the thresholds to obtain our final hand image
    one = cv2.multiply(hue, sat)
    two = cv2.multiply(one, val)
    hands = cv2.multiply(two, output1)

    # Smooth + threshold to filter noise
    hands = cv2.blur(hands, (13, 13))
    ret, hands = cv2.threshold(hands, 200, 255, cv2.THRESH_BINARY)

    original[hands == 0] = 0

    # Show the thresholds if debug is enabled
    if DEBUG:
        cv2.imshow('Saturation threshold', sat)
        cv2.imshow('Hue threshold', hue)
        cv2.imshow('Val threshold', val)
        cv2.imshow('Depth threshold', depThresh * 100)
        cv2.imshow('Hands', original)
    
    # Find the hands by selecting the two largest blobs
    handContours = findHands(hands)
    rects = []
    factor = 20

    # Loop over each of the two blobs
    for j in range(len(handContours)):
        handCnt = handContours[j]

        # Bound a rectangle to the blob
        # Note that we make the rectangle a bit larger to make camshift work a little better
        rect = cv2.boundingRect(handCnt)
        newRect = (max(rect[0] - factor, 0), max(rect[1] - factor, 0), min(rect[0] + rect[2] + factor, shp[1]), min(rect[1] + rect[3] + factor, shp[0]))

        # As long as the area of the rectangle meets a threshold, and it is not too similar to another, append it
        # if debug is enabled, we draw the rectangle
        if (area(newRect) > 2000 and (len(rects) == 0 or not sameRect(newRect, rects[0]))):
            rects.append(newRect)
            if DEBUG:
                cv2.rectangle(originalImage, (max(rect[0] - factor, 0), max(rect[1] - factor, 0)), (min(rect[0] + rect[2] + factor, shp[1]), min(rect[1] + rect[3] + factor, shp[0])), (255, 255, 0))

    # Initialize a camshift tracker for each hand.
    # Note that if one is already tracking, initialize does nothing useful
    rects.sort()
    for j in range(len(rects)):

        # Filter out all pixels outside the rectangle found above
        im = np.zeros(shp + (3,), dtype = "uint8")
        im[rects[j][1] : rects[j][3], rects[j][0] : rects[j][2], :] = 1
        out = cv2.multiply(image, im)
        val = j
        if (len(rects) == 1 and not camShifter[val].tracking):
            val = (0 if rects[0][0] < 270 else 1)
            
        camShifter[val].initialize(original, out, rects[j], colors[val], meanVal)

    if DEBUG:
        print "#### Rects ####"
        print rects
        print "#### ####"

    # For each of the camshift trackers, run the algorithm (does nothing if there is no selection)
    for j in range(len(camShifter)):
        camShift = camShifter[j]
        camShift.run()

        # If we retrieved something useful, draw the ellipse and print information
        if (len(rects) > 0 and (camShift.tracking or camShift.reDraw)):
            try: cv2.ellipse(originalImage, camShift.track_box, camShift.color, 2)
            except: 
                print ""
            if DEBUG:
                print "#### RotatedRect TrackBox 1 ####"
                try: print camShifter[j].track_box
                except: print "Could not print trackbox"
                print "#### ####"

            whichHand = "Left" if j == 1 else "Right"
            print "#### {hand} Hand ####".format(hand = whichHand)
            try: 
                point = worldPoint(camShift.track_box[0][0], camShift.track_box[0][1], depth[camShift.track_box[0][1], camShift.track_box[0][0]])
                print "Position: ", camShift.track_box[0]
                print "Size: ", camShift.track_box[1]
                print "Orientation: ", camShift.track_box[2]
                print "World Point: ", point

            except: print ""
            print "#### ####"

    # Store the points to return
    handList = [None, None]
    for j in range(NUM_HANDS):
        if (camShifter[j].track_box is not None):
            handList[(j + 1) % 2] = camShifter[j].track_box[0]

    return handList, camShifter, colors, originalImage