def process(self, inframe, outframe): jevois.LINFO("process with usb") # Get the next camera image (may block until it is captured): inimg = inframe.get() jevois.LINFO("Input image is {} {}x{}".format(jevois.fccstr(inimg.fmt), inimg.width, inimg.height)) # Get the next available USB output image: outimg = outframe.get() jevois.LINFO("Output image is {} {}x{}".format(jevois.fccstr(outimg.fmt), outimg.width, outimg.height)) # Example of getting pixel data from the input and copying to the output: jevois.paste(inimg, outimg, 0, 0) # We are done with the input image: inframe.done() # Example of in-place processing: jevois.hFlipYUYV(outimg) # Example of simple drawings: jevois.drawCircle(outimg, int(outimg.width/2), int(outimg.height/2), int(outimg.height/4), 2, jevois.YUYV.White) jevois.writeText(outimg, "Hi from Python - @MODULE@", 20, 20, jevois.YUYV.White, jevois.Font.Font10x20) # We are done with the output, ready to send it to host over USB: outframe.send() # Send a string over serial (e.g., to an Arduino). Remember to tell the JeVois Engine to display those messages, # as they are turned off by default. For example: 'setpar serout All' in the JeVois console: jevois.sendSerial("DONE frame {}".format(self.frame)); self.frame += 1
def process(self, inframe, outframe): inimg = inframe.get() self.timer.start() imgbgr = jevois.convertToCvBGR(inimg) h, w, chans = imgbgr.shape outimg = outframe.get() outimg.require("output", w, h + 12, jevois.V4L2_PIX_FMT_YUYV) jevois.paste(inimg, outimg, 0, 0) jevois.drawFilledRect(outimg, 0, h, outimg.width, outimg.height - h, jevois.YUYV.Black) inframe.done() cube = self.detect(imgbgr, outimg) # Load camera calibration if needed: # if not hasattr(self, 'camMatrix'): self.loadCameraCalibration(w, h) if cube is not None: jevois.sendSerial(cube.toJson()) # Write frames/s info from our timer into the edge map (NOTE: does not account for output conversion time): fps = self.timer.stop() jevois.writeText(outimg, fps, 3, h - 10, jevois.YUYV.White, jevois.Font.Font6x10) outframe.send()
def printOnImage(self, outimg, cube, width, height): if outimg is not None and outimg.valid() and cube is not None: jevois.drawRect(outimg, cube.x, cube.y, cube.w, cube.h, 2, jevois.YUYV.MedPurple) jevois.writeText( outimg, "Angle: " + str(int(numpy.degrees(cube.angle))) + " Distance: " + str(int(cube.distance)) + " Pixel width: " + str(cube.w), 3, height + 1, jevois.YUYV.White, jevois.Font.Font6x10)
def process(self, inframe, outframe): # Get the next camera image (may block until it is captured). To avoid wasting much time assembling a composite # output image with multiple panels by concatenating numpy arrays, in this module we use raw YUYV images and # fast paste and draw operations provided by JeVois on those images: inimg = inframe.get() # Start measuring image processing time: self.timer.start() # Convert input image to BGR24: imgbgr = jevois.convertToCvBGR(inimg) h, w, chans = imgbgr.shape # Get pre-allocated but blank output image which we will send over USB: outimg = outframe.get() outimg.require("output", w * 2, h + 12, jevois.V4L2_PIX_FMT_YUYV) jevois.paste(inimg, outimg, 0, 0) jevois.drawFilledRect(outimg, 0, h, outimg.width, outimg.height - h, jevois.YUYV.Black) # Let camera know we are done using the input image: inframe.done() # Get a list of quadrilateral convex hulls for all good objects: hlist = self.detect(imgbgr, outimg) # Load camera calibration if needed: if not hasattr(self, 'camMatrix'): self.loadCameraCalibration(w, h) # Map to 6D (inverse perspective): (rvecs, tvecs) = self.estimatePose(hlist) # Send all serial messages: self.sendAllSerial(w, h, hlist, rvecs, tvecs) # Draw all detections in 3D: self.drawDetections(outimg, hlist, rvecs, tvecs) # Write frames/s info from our timer into the edge map (NOTE: does not account for output conversion time): fps = self.timer.stop() jevois.writeText(outimg, fps, 3, h - 10, jevois.YUYV.White, jevois.Font.Font6x10) # We are done with the output, ready to send it to host over USB: outframe.send()
def detect(self, imgbgr, outimg=None): maxn = 5 # max number of objects we will consider h, w, chans = imgbgr.shape # Convert input image to HSV: imghsv = cv2.cvtColor(imgbgr, cv2.COLOR_BGR2HSV) # Isolate pixels inside our desired HSV range: imgth = cv2.inRange(imghsv, self.HSVmin, self.HSVmax) str = "H={}-{} S={}-{} V={}-{} ".format(self.HSVmin[0], self.HSVmax[0], self.HSVmin[1], self.HSVmax[1], self.HSVmin[2], self.HSVmax[2]) # Create structuring elements for morpho maths: if not hasattr(self, 'erodeElement'): self.erodeElement = cv2.getStructuringElement( cv2.MORPH_RECT, (2, 2)) self.dilateElement = cv2.getStructuringElement( cv2.MORPH_RECT, (2, 2)) # Apply morphological operations to cleanup the image noise: imgth = cv2.erode(imgth, self.erodeElement) imgth = cv2.dilate(imgth, self.dilateElement) # Detect objects by finding contours: im2, contours, hierarchy = cv2.findContours(imgth, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE) str += "N={} ".format(len(contours)) # Only consider the 5 biggest objects by area: contours = sorted(contours, key=cv2.contourArea, reverse=True)[:maxn] hlist = [] # list of hulls of good objects, which we will return str2 = "" beststr2 = "" # Identify the "good" objects: for c in contours: # Keep track of our best detection so far: if len(str2) > len(beststr2): beststr2 = str2 str2 = "" # Compute contour area: area = cv2.contourArea(c, oriented=False) # Compute convex hull: rawhull = cv2.convexHull(c, clockwise=True) rawhullperi = cv2.arcLength(rawhull, closed=True) hull = cv2.approxPolyDP(rawhull, epsilon=self.epsilon * rawhullperi * 3.0, closed=True) # Is it the right shape? if (hull.shape != (4, 1, 2)): continue # 4 vertices for the rectangular convex outline (shows as a trapezoid) str2 += "H" # Hull is quadrilateral huarea = cv2.contourArea(hull, oriented=False) if huarea < self.hullarea[0] or huarea > self.hullarea[1]: continue str2 += "A" # Hull area ok hufill = area / huarea * 100.0 if hufill > self.hullfill: continue str2 += "F" # Fill is ok # Check object shape: peri = cv2.arcLength(c, closed=True) approx = cv2.approxPolyDP(c, epsilon=self.epsilon * peri, closed=True) if len(approx) < 7 or len(approx) > 9: continue # 8 vertices for a U shape str2 += "S" # Shape is ok # Compute contour serr: serr = 100.0 * cv2.matchShapes(c, approx, cv2.CONTOURS_MATCH_I1, 0.0) if serr > self.ethresh: continue str2 += "E" # Shape error is ok # Reject the shape if any of its vertices gets within the margin of the image bounds. This is to avoid # getting grossly incorrect 6D pose estimates as the shape starts getting truncated as it partially exits # the camera field of view: reject = 0 for v in c: if v[0, 0] < self.margin or v[0, 0] >= w - self.margin or v[ 0, 1] < self.margin or v[0, 1] >= h - self.margin: reject = 1 break if reject == 1: continue str2 += "M" # Margin ok # Re-order the 4 points in the hull if needed: In the pose estimation code, we will assume vertices ordered # as follows: # # 0| |3 # | | # | | # 1----------2 # v10+v23 should be pointing outward the U more than v03+v12 is: v10p23 = complex( hull[0][0, 0] - hull[1][0, 0] + hull[3][0, 0] - hull[2][0, 0], hull[0][0, 1] - hull[1][0, 1] + hull[3][0, 1] - hull[2][0, 1]) len10p23 = abs(v10p23) v03p12 = complex( hull[3][0, 0] - hull[0][0, 0] + hull[2][0, 0] - hull[1][0, 0], hull[3][0, 1] - hull[0][0, 1] + hull[2][0, 1] - hull[1][0, 1]) len03p12 = abs(v03p12) # Vector from centroid of U shape to centroid of its hull should also point outward of the U: momC = cv2.moments(c) momH = cv2.moments(hull) vCH = complex( momH['m10'] / momH['m00'] - momC['m10'] / momC['m00'], momH['m01'] / momH['m00'] - momC['m01'] / momC['m00']) lenCH = abs(vCH) if len10p23 < 0.1 or len03p12 < 0.1 or lenCH < 0.1: continue str2 += "V" # Shape vectors ok good = (v10p23.real * vCH.real + v10p23.imag * vCH.imag) / (len10p23 * lenCH) bad = (v03p12.real * vCH.real + v03p12.imag * vCH.imag) / (len03p12 * lenCH) # We reject upside-down detections as those are likely to be spurious: if vCH.imag >= -2.0: continue str2 += "U" # U shape is upright # Fixup the ordering of the vertices if needed: if bad > good: hull = np.roll(hull, shift=1, axis=0) # This detection is a keeper: str2 += " OK" hlist.append(hull) if len(str2) > len(beststr2): beststr2 = str2 # Display any results requested by the users: if outimg is not None and outimg.valid(): if (outimg.width == w * 2): jevois.pasteGreyToYUYV(imgth, outimg, w, 0) jevois.writeText(outimg, str + beststr2, 3, h + 1, jevois.YUYV.White, jevois.Font.Font6x10) return hlist
def process(self, inframe, outframe): # Get the next camera image (may block until it is captured). To avoid wasting much time assembling a composite # output image with multiple panels by concatenating numpy arrays, in this module we use raw YUYV images and # fast paste and draw operations provided by JeVois on those images: inimg = inframe.get() # Start measuring image processing time: self.timer.start() # Convert input image to BGR24: imgbgr = jevois.convertToCvBGR(inimg) h, w, chans = imgbgr.shape # Get pre-allocated but blank output image which we will send over USB: outimg = outframe.get() outimg.require("output", w * 2, h + 12, jevois.V4L2_PIX_FMT_YUYV) #outimg.require("output", w, h + 12, jevois.V4L2_PIX_FMT_YUYV) jevois.paste(inimg, outimg, 0, 0) jevois.drawFilledRect(outimg, 0, h, outimg.width, outimg.height - h, jevois.YUYV.Black) # Let camera know we are done using the input image: inframe.done() # Get a list of quadrilateral convex hulls for all good objects: hlist = self.detect(imgbgr, outimg) # Load camera calibration if needed: if not hasattr(self, 'camMatrix'): self.loadCameraCalibration(w, h) # Map to 6D (inverse perspective): (rvecs, tvecs) = self.estimatePose(hlist) # Average Values for i in range(len(tvecs)): for k in range(len(tvecs[i])): self.tsum[k].append(tvecs[i][k]) while (len(self.tsum[k]) > 10): self.tsum[k].pop(0) for i in range(len(rvecs)): for k in range(len(rvecs[i])): self.rsum[k].append(rvecs[i][k]) while (len(self.rsum[k]) > 10): self.rsum[k].pop(0) # Find distance along ground to robot (Y) try: X = sum(self.tsum[0]) / len(self.tsum[0]) * self.mToFt Y = sum(self.tsum[2]) / len(self.tsum[2]) * self.mToFt Z = sum(self.tsum[1]) / len(self.tsum[1]) * self.mToFt groundDis = -0.2509 + 1.2073 * math.cos( self.cameraAngle - math.atan(Z / Y)) * math.sqrt(math.pow(Z, 2) + math.pow(Y, 2)) except: groundDis = 0 X = 0 Y = 0 Z = 0 # Output Average of Target in Feet and Radians jevois.writeText( outimg, "X: {} Y: {} Angle: {}".format( X, groundDis, (180 / 3.14) * sum(self.rsum[1]) / len(self.rsum[1])), 3, 0, jevois.YUYV.White, jevois.Font.Font6x10) # Send all serial messages: self.sendAllSerial(w, h, hlist, rvecs, tvecs) # Draw all detections in 3D: self.drawDetections(outimg, hlist, rvecs, tvecs) # Write frames/s info from our timer into the edge map (NOTE: does not account for output conversion time): fps = self.timer.stop() jevois.writeText(outimg, fps, 3, h - 10, jevois.YUYV.White, jevois.Font.Font6x10) # Test Serial Output #jevois.sendSerial("{} {}". # format(-1 * self.mToFt, -1 * self.mToFt)) # We are done with the output, ready to send it to host over USB: outframe.send()
def drawDetections(self, outimg, hlist, rvecs=None, tvecs=None): # Show trihedron and parallelepiped centered on object: hw = self.owm * 0.5 hh = self.ohm * 0.5 dd = -max(hw, hh) i = 0 empty = np.array([(0.0), (0.0), (0.0)]) for obj in hlist: # skip those for which solvePnP failed: if np.array_equal(rvecs[i], empty): i += 1 continue jevois.writeText(outimg, str(i), 3, 100 + 10 * i, jevois.YUYV.White, jevois.Font.Font6x10) # Project axis points: axisPoints = np.array([(0.0, 0.0, 0.0), (hw, 0.0, 0.0), (0.0, hh, 0.0), (0.0, 0.0, dd)]) imagePoints, jac = cv2.projectPoints(axisPoints, rvecs[i], tvecs[i], self.camMatrix, self.distCoeffs) # Draw axis lines: jevois.drawLine(outimg, int(imagePoints[0][0, 0] + 0.5), int(imagePoints[0][0, 1] + 0.5), int(imagePoints[1][0, 0] + 0.5), int(imagePoints[1][0, 1] + 0.5), 2, jevois.YUYV.MedPurple) jevois.drawLine(outimg, int(imagePoints[0][0, 0] + 0.5), int(imagePoints[0][0, 1] + 0.5), int(imagePoints[2][0, 0] + 0.5), int(imagePoints[2][0, 1] + 0.5), 2, jevois.YUYV.MedGreen) # Normal to plane jevois.drawLine(outimg, int(imagePoints[0][0, 0] + 0.5), int(imagePoints[0][0, 1] + 0.5), int(imagePoints[3][0, 0] + 0.5), int(imagePoints[3][0, 1] + 0.5), 2, jevois.YUYV.MedGrey) # Also draw a parallelepiped: cubePoints = np.array([(-hw, -hh, 0.0), (hw, -hh, 0.0), (hw, hh, 0.0), (-hw, hh, 0.0), (-hw, -hh, dd), (hw, -hh, dd), (hw, hh, dd), (-hw, hh, dd)]) cu, jac2 = cv2.projectPoints(cubePoints, rvecs[i], tvecs[i], self.camMatrix, self.distCoeffs) # Round all the coordinates and cast to int for drawing: cu = np.rint(cu) # Draw parallelepiped lines: jevois.drawLine(outimg, int(cu[0][0, 0]), int(cu[0][0, 1]), int(cu[1][0, 0]), int(cu[1][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[1][0, 0]), int(cu[1][0, 1]), int(cu[2][0, 0]), int(cu[2][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[2][0, 0]), int(cu[2][0, 1]), int(cu[3][0, 0]), int(cu[3][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[3][0, 0]), int(cu[3][0, 1]), int(cu[0][0, 0]), int(cu[0][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[4][0, 0]), int(cu[4][0, 1]), int(cu[5][0, 0]), int(cu[5][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[5][0, 0]), int(cu[5][0, 1]), int(cu[6][0, 0]), int(cu[6][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[6][0, 0]), int(cu[6][0, 1]), int(cu[7][0, 0]), int(cu[7][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[7][0, 0]), int(cu[7][0, 1]), int(cu[4][0, 0]), int(cu[4][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[0][0, 0]), int(cu[0][0, 1]), int(cu[4][0, 0]), int(cu[4][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[1][0, 0]), int(cu[1][0, 1]), int(cu[5][0, 0]), int(cu[5][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[2][0, 0]), int(cu[2][0, 1]), int(cu[6][0, 0]), int(cu[6][0, 1]), 1, jevois.YUYV.LightGreen) jevois.drawLine(outimg, int(cu[3][0, 0]), int(cu[3][0, 1]), int(cu[7][0, 0]), int(cu[7][0, 1]), 1, jevois.YUYV.LightGreen) i += 1
def detect(self, imgbgr, outimg=None): maxn = 9 # max number of objects we will consider h, w, chans = imgbgr.shape # Convert input image to HSV: imghsv = cv2.cvtColor(imgbgr, cv2.COLOR_BGR2HSV) # Isolate pixels inside our desired HSV range: imgth = cv2.inRange(imghsv, self.HSVmin, self.HSVmax) outstr = "H={}-{} S={}-{} V={}-{} ".format( self.HSVmin[0], self.HSVmax[0], self.HSVmin[1], self.HSVmax[1], self.HSVmin[2], self.HSVmax[2]) # Create structuring elements for morpho maths: if not hasattr(self, 'erodeElement'): self.erodeElement = cv2.getStructuringElement( cv2.MORPH_RECT, (2, 2)) self.dilateElement = cv2.getStructuringElement( cv2.MORPH_RECT, (2, 2)) # Apply morphological operations to cleanup the image noise: imgth = cv2.erode(imgth, self.erodeElement) imgth = cv2.dilate(imgth, self.dilateElement) contours, hierarchy = cv2.findContours(imgth, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE) contours = sorted(contours, key=cv2.contourArea, reverse=True)[:maxn] # Find and create the raw hulls hulls = () centers = () for i in range(len(contours)): rawhull = cv2.convexHull(contours[i], clockwise=True) rawhullperi = cv2.arcLength(rawhull, closed=True) hull = cv2.approxPolyDP(rawhull, epsilon=self.epsilon * rawhullperi * 3.0, closed=True) # Outline hull and find center huarea = cv2.contourArea(hull, oriented=False) if len(hull) == 4 and huarea > self.hullarea[ 0] and huarea < self.hullarea[1]: npHull = np.array(hull, dtype=int).reshape(len(hull), 2, 1) centers += (((npHull[0, 0, 0] + npHull[1, 0, 0] + npHull[2, 0, 0] + npHull[3, 0, 0]) / 4, (npHull[0, 1, 0] + npHull[1, 1, 0] + npHull[2, 1, 0] + npHull[3, 1, 0]) / 4), ) hulls += (npHull, ) # Reset image imgth = cv2.inRange(imghsv, np.array([122, 122, 122], dtype=np.uint8), np.array([122, 122, 122], dtype=np.uint8)) # Finds the two lowest points of each hull, typically: bottom left point, bottom right point bottomPoints = () for i in range(len(hulls)): bottomPoint = (-1, 0) secondPoint = (-1, 0) for k in range(len(hulls[i])): if (bottomPoint[0] == -1): bottomPoint = (k, hulls[i][k, 1, 0]) elif (bottomPoint[1] < hulls[i][k, 1, 0]): bottomPoint = (k, hulls[i][k, 1, 0]) for k in range(len(hulls[i])): if (k != bottomPoint[0]): if (secondPoint[0] == -1): secondPoint = (k, hulls[i][k, 1, 0]) elif (secondPoint[1] < hulls[i][k, 1, 0]): secondPoint = (k, hulls[i][k, 1, 0]) if (abs(centers[i][0] - hulls[i][bottomPoint[0], 0, 0]) > abs(centers[i][0] - hulls[i][secondPoint[0], 0, 0])): bottomPoints += ((i, bottomPoint[0]), ) else: bottomPoints += ((i, secondPoint[0]), ) # Find closest other hull to each hull nearHull = () for i in range(len(hulls)): closest = (-1, 0.0) for k in range(len(hulls)): if (i != k): if ((hulls[bottomPoints[i][0]][bottomPoints[i][1], 0, 0] - centers[i][0]) * (hulls[bottomPoints[k][0]][bottomPoints[k][1], 0, 0] - centers[k][0])) < 0: if (centers[i][0] < centers[k][0] and hulls[bottomPoints[i][0]][bottomPoints[i][1], 0, 0] < centers[i][0] ) or (centers[i][0] > centers[k][0] and hulls[bottomPoints[i][0]] [bottomPoints[i][1], 0, 0] > centers[i][0]): if (closest[0] == -1): closest = ( k, math.pow(centers[i][0] - centers[k][0], 2) + math.pow(centers[i][1] - centers[k][1], 2)) elif ( closest[1] > (math.pow(centers[i][0] - centers[k][0], 2) + math.pow(centers[i][1] - centers[k][1], 2))): closest = ( k, math.pow(centers[i][0] - centers[k][0], 2) + math.pow(centers[i][1] - centers[k][1], 2)) nearHull += (closest[0], ) # Find the two closest points between a hull and its nearest hull closePoint = () for i in range(len(hulls)): closest = (-1, -1, 0.0) for k in range(len(hulls[i])): for j in range(len(hulls[nearHull[i]])): if (closest[0] == -1 and closest[1] == -1): closest = (k, j, math.pow( hulls[i][k, 0, 0] - hulls[nearHull[i]][j, 0, 0], 2) + math.pow( hulls[i][k, 1, 0] - hulls[nearHull[i]][j, 1, 0], 2)) elif (closest[2] > math.pow( hulls[i][k, 0, 0] - hulls[nearHull[i]][j, 0, 0], 2) + math.pow( hulls[i][k, 1, 0] - hulls[nearHull[i]][j, 1, 0], 2)): closest = (k, j, math.pow( hulls[i][k, 0, 0] - hulls[nearHull[i]][j, 0, 0], 2) + math.pow( hulls[i][k, 1, 0] - hulls[nearHull[i]][j, 1, 0], 2)) closePoint += ((closest[0], closest[1]), ) # Find the target center hullCenter = [] for i in range(len(hulls)): if (centers[i][0] > centers[nearHull[i]][0] and nearHull[nearHull[i]] == i): hullCenter += [(centers[i][0] + centers[nearHull[i]][0]) / 2, (centers[i][1] + centers[nearHull[i]][1]) / 2, i], # Choose target closest to center of screen targetHull = (-1, 0) for i in range(len(hullCenter)): if (targetHull[0] == -1): targetHull = (hullCenter[i][2], abs(hullCenter[i][0] - self.width / 4)) elif (targetHull[1] > abs(hullCenter[i][0] - self.width / 4)): targetHull = (hullCenter[i][2], abs(hullCenter[i][0] - self.width / 4)) # Generate the U-Shape map of the target for pose estimation if a target exists hlist = [] if (targetHull[0] != -1): # Calculates the displacement of the edge of the physical target to the corner of the drawn target xChange = hulls[nearHull[targetHull[0]]][ (closePoint[targetHull[0]][1] + 3) % 4, 0, 0] - hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 1) % 4, 0, 0] yChange = hulls[nearHull[targetHull[0]]][ (closePoint[targetHull[0]][1] + 3) % 4, 1, 0] - hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 1) % 4, 1, 0] # Map points to desired U-Shape # # 0 3 # | | # | | # | | # 1_______________2 # corners = ( (hulls[nearHull[targetHull[0]]][(closePoint[targetHull[0]][1] + 1) % 4, 0, 0], hulls[nearHull[targetHull[0]]][(closePoint[targetHull[0]][1] + 1) % 4, 1, 0]), (hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 1) % 4, 0, 0] + xChange * self.targetRatio, hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 1) % 4, 1, 0] + yChange * self.targetRatio), (hulls[nearHull[targetHull[0]]][ (closePoint[targetHull[0]][1] + 3) % 4, 0, 0] - xChange * self.targetRatio, hulls[nearHull[targetHull[0]]][ (closePoint[targetHull[0]][1] + 3) % 4, 1, 0] - yChange * self.targetRatio), (hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 3) % 4, 0, 0], hulls[targetHull[0]][(closePoint[targetHull[0]][0] + 3) % 4, 1, 0]), ) # Maps U-Shape's corners weighted by Rectangular Corners poly = np.array([ [int(corners[0][0]), int(corners[0][1])], [int(corners[1][0]), int(corners[1][1])], [int(corners[2][0]), int(corners[2][1])], [int(corners[3][0]), int(corners[3][1])], [ int((corners[3][0] * (1 - self.percentFill) + corners[0][0] * (self.percentFill))), int((corners[3][1] * (1 - self.percentFill) + corners[0][1] * (self.percentFill))) ], [ int((corners[2][0] * (1 - self.percentFill) + corners[0][0] * (self.percentFill))), int((corners[2][1] * (1 - self.percentFill) + corners[0][1] * (self.percentFill))) ], [ int((corners[1][0] * (1 - self.percentFill) + corners[3][0] * (self.percentFill))), int((corners[1][1] * (1 - self.percentFill) + corners[3][1] * (self.percentFill))) ], [ int((corners[0][0] * (1 - self.percentFill) + corners[3][0] * (self.percentFill))), int((corners[0][1] * (1 - self.percentFill) + corners[3][1] * (self.percentFill))) ], ], np.int32) # Draw U-Shaped Map poly = poly.reshape((-1, 1, 2)) cv2.fillPoly(imgth, [poly], (255, 255, 255)) # Does not pass if the sides of the rectangle are too close to the sides of the image isInRange = True for point in corners: if (point[0] > self.pxThreshold and point[0] < self.width - self.pxThreshold and point[1] > self.pxThreshold and point[1] < self.height - self.pxThreshold): continue isInRange = False if (isInRange): hlist.append(corners) # Display any results requested by the users: if outimg is not None and outimg.valid(): if (outimg.width == w * 2): jevois.pasteGreyToYUYV(imgth, outimg, w, 0) jevois.writeText(outimg, "yeet 2.0", 3, h + 1, jevois.YUYV.White, jevois.Font.Font6x10) # Return the target return hlist
def detect(self, imggray, outimg=None): h, w = imggray.shape hlist = [] # Create a keypoint detector if needed: if not hasattr(self, 'detector'): self.detector = cv2.ORB_create() # Load training image and detect keypoints on it if needed: if not hasattr(self, 'refkp'): refimg = cv2.imread(self.fname, 0) self.refkp, self.refdes = self.detector.detectAndCompute( refimg, None) # Also store corners of reference image and of window for homography mapping: refh, refw = refimg.shape self.refcorners = np.float32([[0.0, 0.0], [0.0, refh], [refw, refh], [refw, 0.0]]).reshape(-1, 1, 2) self.wincorners = np.float32( [[ self.winleft * refw / self.owm, self.wintop * refh / self.ohm ], [ self.winleft * refw / self.owm, (self.wintop + self.winh) * refh / self.ohm ], [(self.winleft + self.winw) * refw / self.owm, (self.wintop + self.winh) * refh / self.ohm], [(self.winleft + self.winw) * refw / self.owm, self.wintop * refh / self.ohm]]).reshape(-1, 1, 2) jevois.LINFO( "Extracted {} keypoints and descriptors from {}".format( len(self.refkp), self.fname)) # Compute keypoints and descriptors: kp, des = self.detector.detectAndCompute(imggray, None) str = "{} keypoints".format(len(kp)) # Create a matcher if needed: if not hasattr(self, 'matcher'): self.matcher = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) # Compute matches between reference image and camera image, then sort them by distance: matches = self.matcher.match(des, self.refdes) matches = sorted(matches, key=lambda x: x.distance) str += ", {} matches".format(len(matches)) # Keep only good matches: lastidx = 0 for m in matches: if m.distance < self.distth: lastidx += 1 else: break matches = matches[0:lastidx] str += ", {} good".format(len(matches)) # If we have enough matches, compute homography: corners = [] wincorners = [] if len(matches) >= 10: obj = [] scene = [] # Localize the object (see JeVois C++ class ObjectMatcher for details): for m in matches: obj.append(self.refkp[m.trainIdx].pt) scene.append(kp[m.queryIdx].pt) # compute the homography hmg, mask = cv2.findHomography(np.array(obj), np.array(scene), cv2.RANSAC, 5.0) # Check homography conditioning using SVD: u, s, v = np.linalg.svd(hmg, full_matrices=False) # We need the smallest eigenvalue to not be too small, and the ratio of largest to smallest eigenvalue to be # quite large for our homography to be declared good here. Note that linalg.svd returns the eigenvalues in # descending order already: if s[-1] > 0.001 and s[0] / s[-1] > 100: # Project the reference image corners to the camera image: corners = cv2.perspectiveTransform(self.refcorners, hmg) wincorners = cv2.perspectiveTransform(self.wincorners, hmg) # Display any results requested by the users: if outimg is not None and outimg.valid(): if len(corners) == 4: jevois.drawLine(outimg, int(corners[0][0, 0] + 0.5), int(corners[0][0, 1] + 0.5), int(corners[1][0, 0] + 0.5), int(corners[1][0, 1] + 0.5), 2, jevois.YUYV.LightPink) jevois.drawLine(outimg, int(corners[1][0, 0] + 0.5), int(corners[1][0, 1] + 0.5), int(corners[2][0, 0] + 0.5), int(corners[2][0, 1] + 0.5), 2, jevois.YUYV.LightPink) jevois.drawLine(outimg, int(corners[2][0, 0] + 0.5), int(corners[2][0, 1] + 0.5), int(corners[3][0, 0] + 0.5), int(corners[3][0, 1] + 0.5), 2, jevois.YUYV.LightPink) jevois.drawLine(outimg, int(corners[3][0, 0] + 0.5), int(corners[3][0, 1] + 0.5), int(corners[0][0, 0] + 0.5), int(corners[0][0, 1] + 0.5), 2, jevois.YUYV.LightPink) jevois.writeText(outimg, str, 3, h + 4, jevois.YUYV.White, jevois.Font.Font6x10) # Return window corners if we did indeed detect the object: hlist = [] if len(wincorners) == 4: hlist.append(wincorners) return hlist