def open_camera(self): UITool.msg_window('Info', 'Start face distant protect mode.') flag = True vs = WebcamVideoStream(src=0) vs.start() time.sleep(2.0) cv2.startWindowThread() while flag: check = True frame = vs.read() frame = imutils.resize(frame, width=800) display_frame = frame.copy() capt_faces = FaceCapture.cap(frame) if capt_faces.has_face: display_frame = ImgTool.add_face_block(display_frame, capt_faces) face_data = FacesProc.get_all_faces_with_encode_low_filter( frame, self.mode) if face_data: check = self.check_face_encode(face_data) print(f"status: {check}") if not check: print( f" WARN - find stranger and no master in the same time !!!" ) self.alert = True else: print('fetched face data is not enough') if self.display: display_frame = cv2.flip(display_frame, 1) if check: text = 'Safe' print(text) display_frame = ImgTool.add_text(display_frame, text) else: text = 'Detect Stranger!' display_frame = ImgTool.add_text(display_frame, text, color='red') cv2.imshow('Frame', display_frame) key = cv2.waitKey(1) if self.alert or key == 27 or key == ord('q'): flag = False cv2.waitKey(500) cv2.destroyAllWindows() cv2.waitKey(500) vs.stop() cv2.destroyAllWindows()
def launch(self): flag = True vs = WebcamVideoStream(src=0) vs.start() time.sleep(2.0) fetch_face = 0 last_face_ts = 0 cv2.startWindowThread() while flag: frame = vs.read() if frame is None: continue frame = imutils.resize(frame, width=800) display_frame = frame.copy() capt_faces = FaceCapture.cap(frame) if capt_faces.face_count == 1: display_frame = ImgTool.add_face_block(display_frame, capt_faces) ts = FileTool.get_curr_ts() if ts - last_face_ts > RecordMD.INTERVAL: if self.__record_face(frame, capt_faces, ts): fetch_face += 1 print( f'fetch face successfully. ({fetch_face}/{self.fetch_count})' ) last_face_ts = ts else: print(f'failed to record faces') if self.display: display_frame = cv2.flip(display_frame, 1) text = f"{fetch_face} / {self.fetch_count}" display_frame = ImgTool.add_text(display_frame, text) cv2.imshow('Frame', display_frame) key = cv2.waitKey(1) if fetch_face == self.fetch_count or key == 27 or key == ord('q'): flag = False cv2.waitKey(500) cv2.destroyAllWindows() cv2.waitKey(500) vs.stop() cv2.destroyAllWindows() # self.__img_meta.to_csv(RecordMD.meta_path, index=False, encoding='utf-8') msg = 'Finish recoding master data.' UITool.msg_window(msg=msg) print('finish saving img meta.')
def from_camera(self, camera_id=0, show=False): """ Detects objects in frames from a camera feed :param camera_id: the ID of the camera in the system """ def get_frame(stream): frame = stream.read() ret = True return ret, frame stream = WebcamVideoStream(src=camera_id) stream.start() self._detect_from_stream(get_frame, stream, show) stream.stop()
def predict(): face_classifier = cv2.CascadeClassifier( "C:/Users/uvss/AppData/Local/Programs/Python/Python37-32/Lib/site-packages/cv2/data/haarcascade_frontalface_default.xml" ) face_recognizer = cv2.face.LBPHFaceRecognizer_create() face_recognizer.read("trainer/training_data.yml") vs = WebcamVideoStream( "rtsp://*****:*****@192.168.1.90/axis-media/media.amp") vs = vs.start() if vs.grabbed: while True: image = vs.read() img = image.copy() gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) faces = face_classifier.detectMultiScale(gray, 1.2, 5) if faces is (): pass for (x, y, w, h) in faces: cv2.rectangle(img, (x, y), (x + w, y + h), (102, 255, 102), 2) roi_gray = gray[y:y + h, x:x + w] label, conf = face_recognizer.predict(roi_gray) print(label, int(conf)) if conf > 85: label = "Unknown" cv2.putText(img, str(label), (x + 5, y + h - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2) cv2.imshow("frames", img) if cv2.waitKey(1) == ord("q"): break vs.stop() cv2.destroyAllWindows()
def training_Data(path): while True: try: details = { "name": input("Provide a name for the Employee:- "), "depart": input("Department:- "), "ID": int(input("Office Id (number) :- ")) } if bool(re.search(r'\d', details["name"])) or bool( re.search(r'\d', details["depart"])): raise ValueError( "[Error] Invalid Input for Name or Department") break except Exception as err: print("[Error] Invalid Input for Office Id") continue sampled_faces = [] sampled_labels = [] img_name = None usr_id, label = 1, 1 img_id = 0 users = os.listdir(path) users.sort( key=lambda x: int(x[1:])) # sorting all the user data directories if users == []: usr_dir = path + "/s" + str(usr_id) + "/" os.mkdir(usr_dir) else: label = int(users[-1][1:]) + 1 usr_id = str(label) # get [1:] part of last element in the list usr_dir = path + "/s" + usr_id + "/" os.mkdir(usr_dir) vs = WebcamVideoStream( src="rtsp://*****:*****@192.168.1.90/axis-media/media.amp") if vs.grabbed: vs = vs.start() while (True): frame = vs.read() cv2.imshow("frame", frame) faces = face_Detector(frame) if cv2.waitKey(1) == ord("q"): # sys.exit() break if faces is None or faces == []: continue for face in faces: img_id += 1 cv2.imshow("face", face) img_name = usr_dir + str(img_id) + ".jpg", face cv2.imwrite(img_name, face) sampled_labels.append(label) sampled_faces.append(face) if img_id >= 50: break vs.stop() cv2.destroyAllWindows() return (sampled_faces, sampled_labels, details)
class WebcamIngestorSource(FrameIngestorSource): """Returns frames from the webcam.""" def __init__(self, source_num=0): """Initialize object. Raises: TypeError: when source_num is not an int Args: source_num: video stream number """ if not isinstance(source_num, int): raise TypeError('`source_num` is not an int') self._source_num = source_num def start(self): """Open the source. Raises: RuntimeError: when could not open stream. """ self.vs = WebcamVideoStream(src=self._source_num) if not self.vs.stream.isOpened(): msg = 'Could not open webcam stream {0}'.format(self._source_num) raise RuntimeError(msg) self.vs.start() self.initialized = True def get_frame(self): """Fetch single frame from source. Returns: fetched frame. Raises: RuntimeError: if source is not initialized. """ if not self.initialized: raise RuntimeError('Source not initialized.') return self.vs.read() def stop(self): """Stop the source and free all allocated resources.""" self.vs.stop()
class videoStream: def __init__(self): print("Initializing video stream") self.stream = WebcamVideoStream(src=0) # Start the video stream self.stream.start() time.sleep(1.0) # wait for stream to initialize print("Video stream initialized") # end init def grabFrame(self): return self.stream.read() # grab a frame and return # end grabFrame def stop(self): self.stream.stop()
class Threaded_Frame: def __init__(self, src): self.stream = WebcamVideoStream(src=src) def start(self): # start the threaded video stream return self.stream.start() def read(self): # return the current frame return self.stream.read() def stop(self): # stop the thread and release any resources self.stream.stop()
def detect_faces(): print("[INFO] starting video stream...") vs = WebcamVideoStream( src="rtsp://*****:*****@192.168.1.90/axis-media/media.amp") if vs.grabbed: vs = vs.start() time.sleep(2.0) print("------started survillancing------") id = 0 x = 1 while (True): x += 1 frame = vs.read() id += 1 if x % 2 == 0: gray = face_detector(frame, id) cv2.imshow('frame', gray) if cv2.waitKey(10) & 0xFF == ord('q'): break vs.stop() cv2.destroyAllWindows()
class VideoStream: def __init__(self, src=0, usePiCamera=False, resolution=(320, 240), framerate=32): # check to see if the picamera module should be used if usePiCamera: # only import the picamera packages unless we are # explicity told to do so -- this helps remove the # requirement of `picamera[array]` from desktops or # laptops that still want to use the `imutils` package from pivideostream import PiVideoStream # initialize the picamera stream and allow the camera # sensor to warmup self.stream = PiVideoStream(resolution=resolution, framerate=framerate) # otherwise, we are using OpenCV so initialize the webcam # stream else: self.stream = WebcamVideoStream(src=src) def start(self): # start the threaded video stream return self.stream.start() def update(self): # grab the next frame from the stream self.stream.update() def read(self): # return the current frame return self.stream.read() def stop(self): # stop the thread and release any resources self.stream.stop()
class Pipeline: # created a *threaded* video stream, allow the camera sensor to warmup, # and start the FPS counter print("[INFO] sampling THREADED frames from webcam...") #Constructor creates a list def __init__(self, fixed_size): self.vs1 = WebcamVideoStream(src=0) #For anylizing self.vs1.start() self.vs2 = WebcamVideoStream(src=0) #For anylizing self.vs2.start() self.cb = CircularBuffer(fixed_size) self.stopped = False self.current_stream = (None, None) self.disp = None self.haptic = None #Start thread def start(self): t = Thread(target=self.capture, args=()) t.daemon = True t.start() return self def stop(self): self.stopped = True def capture(self): # keep looping infinitely # if the thread indicator variable is set, stop the # threa while not self.stopped: left_captured_frame = self.vs1.read() right_captured_frame = self.vs2.read() self.cb.enqueue((left_captured_frame, right_captured_frame)) def push(self): pushed_stereo = self.capture() self.cb.enqueue(pushed_stereo) def pull(self): pulled_stereo = self.cb.dequeue() return pulled_stereo def get_disp(self): plt.clf() self.disp = i2d(self.stereo, 16) plt.imshow(self.disp, 'gray') buf = io.BytesIO() plt.savefig(buf, format='png') buf.seek(0) return buf.read() def get_depth(self): plt.clf() # Then send the disparity to haptic self.haptic = np.array(depth_to_haptic(self.disp)) plt.scatter(self.haptic[:, 0], self.haptic[:, 1]) buf = io.BytesIO() plt.savefig(buf, format='png') buf.seek(0) return buf.read() def get_frame(self): self.stereo = self.pull() # depth = self.process(stereo_feed) left_feed = self.stereo[0] right_feed = self.stereo[1] # We are using Motion JPEG, but OpenCV defaults to capture raw images, # so we must encode it into JPEG in order to correctly display the # video stream. l_ret, l_jpeg = cv2.imencode('.jpg', left_feed) r_ret, r_jpeg = cv2.imencode('.jpg', right_feed) return l_jpeg.tobytes(), r_jpeg.tobytes() def gen(self, dir): while True: self.current_stream = self.get_frame() if dir == 'left': yield (b'--stream_left\r\n' b'Content-Type: image/jpeg\r\n\r\n' + self.current_stream[0] + b'\r\n\r\n') if dir == 'right': yield (b'--stream_right\r\n' b'Content-Type: image/jpeg\r\n\r\n' + self.current_stream[1] + b'\r\n\r\n') if dir == 'disp': self.disp = self.get_disp() yield (b'--stream_right\r\n' b'Content-Type: image/jpeg\r\n\r\n' + self.disp + b'\r\n\r\n') if dir == 'depth': yield (b'--stream_depth\r\n' b'Content-Type: image/jpeg\r\n\r\n' + self.haptic + b'\r\n\r\n')
def main(): # Initialize background substractor # Currently CV2 provide 2 types, "createBackgroundSubtractorKNN" and "createBackgroundSubtractorMOG2" # 1. "createBackgroundSubtractorKNN" # - subtraction is alot cleaner if background color more consistence # 2. "createBackgroundSubtractorMOG2" # - substraction based on Machine Learning (ML), more cleaner if background have too many color for KNN to work. # - Result of substraction is not as complete compare to KNN backSub = cv2.createBackgroundSubtractorKNN(history=450, dist2Threshold=150.0, detectShadows=True) # cap = cv2.VideoCapture('fish4.mp4') #cap = FileVideoStream('fish4.mp4').start() #initialize file video reader cap = WebcamVideoStream(0) cap.start() time.sleep(1.0) # Block for 1 sec, to let "cap" buffer frames frame = cap.read() # Get current frame vh, vw = frame.shape[: 2] # Get current frame height and width, [0]: height, [1]: width vh = int(vh / 2) vw = int(vw / 2) videWriter = cv2.VideoWriter('out.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 30, (vw, vh)) #Initialize video writer # Set default center line line_thickness = 5 # default line thickness to 5px line_center = int( (vw / 2)) # divide 2 with video width (vw) to get video center pixel fishes = [] # store fish id & tracker counter = 0 # fish counter fps = FPS().start() #Initialize FPS counter ttQ = Queue() tt = Process(target=showMe, args=(ttQ, )) tt.daemon = True tt.start() # While cap has more frame continue while True: frame = cv2.resize(frame, (vw, vh), cv2.INTER_AREA) blank_frame = frame.copy( ) # Create a copy of original image for filtering trackers = multiTracker.update(frame) if len(trackers) > 0: for tracker in trackers: c, bbox = tracker # c = id #success, bbox = tracker.update(frame) # run tracker update based on current frame # Should tracker success = false, removed it from array to prevent future processes #if not success: # fishes.pop(idx) # continue # Calculate & Draw offset = 0 xy1 = (int(bbox[0]) - offset, int(bbox[1]) - offset ) # staring X and Y bounding box xy2 = (int(bbox[2]) + offset, int(bbox[3]) + offset ) # Width and Height bounding box # Note: # OpenCV Tracker return Region of Interest (ROI) which consist # 1. starting point X and Y coordinate in pixel # 2. width and height of the bounding box in pixel # OpenCV rectangle function however is draw using 2 set of coordinate in pixel # starting point (Top Left) and end point (Bottom Right) # "xy1" is the starting coordinate (Top Left), as such using # ((xy1[0] + xy2[0]), (xy1[1] + xy2[1])) # we can calculate the 2nd set coordinate (Bottom Right) # Bottom = starting X point + bounding box width # Right = starting Y point + bounding box height cv2.rectangle(frame, xy1, ((xy1[0] + xy2[0]), (xy1[1] + xy2[1])), (0, 0, 255), 1) cv2.putText(frame, 'id: {}'.format(c), (xy1[0], xy1[1] - 10), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 0), 1, cv2.LINE_AA) cv2.rectangle(blank_frame, xy1, ((xy1[0] + xy2[0]), (xy1[1] + xy2[1])), (255, 255, 255), -1) # Note: # OpenCV "findContours" function only work with image in gray color gFrame = cv2.cvtColor(blank_frame, cv2.COLOR_RGB2GRAY) # Convert image to gray fMask = backSub.apply(gFrame) # Apply background seperation algorythm fMask = cv2.morphologyEx(fMask, cv2.MORPH_OPEN, np.ones((5, 5), np.uint8), iterations=2) # Fix deform contour fMask = cv2.morphologyEx(fMask, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8), iterations=2) # Fix deform contour fMask = cv2.bitwise_and(gFrame, gFrame, mask=fMask) # combine targeted frame with mask fMask = cv2.GaussianBlur(fMask, (5, 5), 0) # add blur to further reduce pixel deform ret, thresh = cv2.threshold( fMask, 50, 255, cv2.THRESH_BINARY) # Create threshold algorythm contours, hierarchy = cv2.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # find contours # Loop through all found contours # Should any contour found and is within "center_line" tag and track it for contour in contours: x, y, w, h = cv2.boundingRect(contour) if x + w >= int(line_center) and x + w <= int(line_center + line_thickness): counter += 1 # tracker = cv2.TrackerCSRT_create() # tracker.init(frame, (int(x), int(y), int(w), int(h))) # fishes.append((counter, tracker)) multiTracker.add(counter, frame, (int(x), int(y), int(w), int(h))) # cv2.rectangle(frame, (int(x), int(y)), (int(x + w), int(y + h)), (0,0,255), 1) # Draw "line_center" frame = cv2.rectangle(frame, (line_center, vh), (line_center + line_thickness, 0), (0, 0, 0), -1) # Calculate, Generate and Draw "Total Fish" text # Calculate, Generate and Draw "FPS" text fps.update() fps.stop() tt = (frame, counter, fps, vw, vh, line_thickness) ttQ.put(tt) # Display the final combine of the orignal frame including tracked item # cv2.imshow('frame', frame) # cv2.imshow('mask', fMask) # Display frame is being refresh every 30ms, change to 0 if manual forward required key = cv2.waitKey(30) # should "Q" is press, stop loop if key == ord('q'): break # videWriter.write(frame) # Write frame to video output # ok, frame = cap.read() frame = cap.read()
def main(): height = 300 stream = WebcamVideoStream(0) stream.start() # tracker = Tracker() fps = FPS().start() print("[INFO] Camera warming up . . .") time.sleep(2) cv2.namedWindow("Explorer") cv2.createTrackbar("HueMax", "Explorer", 255, 255, nothing) cv2.createTrackbar("SatMax", "Explorer", 255, 255, nothing) cv2.createTrackbar("ValMax", "Explorer", 255, 255, nothing) cv2.createTrackbar("HueMin", "Explorer", 0, 255, nothing) cv2.createTrackbar("SatMin", "Explorer", 0, 255, nothing) cv2.createTrackbar("ValMin", "Explorer", 0, 255, nothing) cv2.createTrackbar("GrayMin", "Explorer", 60, 200, nothing) upperThresh = np.zeros(3) lowerThresh = np.zeros(3) cannyLowerThresh = 50 GRAY = False while not stream.stopped: # Get new frame frame = stream.read() frame = imutils.resize(frame, height=height) fps.stop() # Get track bar values upperThresh[0] = cv2.getTrackbarPos("HueMax", "Explorer") upperThresh[1] = cv2.getTrackbarPos("SatMax", "Explorer") upperThresh[2] = cv2.getTrackbarPos("ValMax", "Explorer") lowerThresh[0] = cv2.getTrackbarPos("HueMin", "Explorer") lowerThresh[1] = cv2.getTrackbarPos("SatMin", "Explorer") lowerThresh[2] = cv2.getTrackbarPos("ValMin", "Explorer") cannyLowerThresh = cv2.getTrackbarPos("GrayMin", "Explorer") # Gray scale mode and colour mode if GRAY: gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) gray = cv2.bilateralFilter(gray, 11, 17, 17) res1 = cv2.Canny(gray, cannyLowerThresh, 200) else: # Detect Object hsv_mask, res1 = hsv_masking(frame, lowerThresh, upperThresh) # Track Object track_position = hsv_detection(hsv_mask) # Calculate node and draw finger for centroid in track_position: # Draw and label each finger cv2.circle(frame, tuple(centroid), 6, (255, 255, 255), -1) cv2.putText( frame, "(%d, %d)" % (*centroid, ), (abs(centroid[0] - 25), abs(centroid[1] - 25)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2, ) cv2.putText( res1, "Press G to toggle greyscale canny", (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2, ) cv2.putText( res1, "Drag the slide bars to explore the HSV colour space", (10, height - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2, ) cv2.putText( frame, datetime.datetime.now().strftime("%A %d %B %Y %I:%M:%S%p") + " Frame %d" % fps._numFrames, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2, ) # Update the frames cv2.imshow("Live Feed", frame) cv2.imshow("Explorer", res1) # Keyboard OP k = cv2.waitKey(5) & 0xFF if k == 27 or k == ord("q"): # Esc break elif k == ord("g"): # toggle gray GRAY = not GRAY fps.update() # Stops video stream, calculates FPS and does some clean ups fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) stream.stop() cv2.destroyAllWindows()
class EyeCanSee(object): def __init__(self, center=int(cvsettings.CAMERA_WIDTH / 2), debug=False, is_usb_webcam=True, period_s=0.025): # Our video stream # If its not a usb webcam then get pi camera if not is_usb_webcam: self.vs = PiVideoStream(resolution=(cvsettings.CAMERA_WIDTH, cvsettings.CAMERA_HEIGHT)) # Camera cvsettings self.vs.camera.shutter_speed = cvsettings.SHUTTER self.vs.camera.exposure_mode = cvsettings.EXPOSURE_MODE self.vs.camera.exposure_compensation = cvsettings.EXPOSURE_COMPENSATION self.vs.camera.awb_gains = cvsettings.AWB_GAINS self.vs.camera.awb_mode = cvsettings.AWB_MODE self.vs.camera.saturation = cvsettings.SATURATION self.vs.camera.rotation = cvsettings.ROTATION self.vs.camera.video_stabilization = cvsettings.VIDEO_STABALIZATION self.vs.camera.iso = cvsettings.ISO self.vs.camera.brightness = cvsettings.BRIGHTNESS self.vs.camera.contrast = cvsettings.CONTRAST # Else get the usb camera else: self.vs = WebcamVideoStream(src=0) self.vs.stream.set(cv2.CAP_PROP_FRAME_WIDTH, cvsettings.CAMERA_WIDTH) self.vs.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, cvsettings.CAMERA_HEIGHT) # Has camera started self.camera_started = False self.start_camera() # Starts our camera # To calculate our error in positioning self.center = center # To determine if we actually detected lane or not self.detected_lane = False # debug mode on? (to display processed images) self.debug = debug # Time interval between in update (in ms) # FPS = 1/period_s self.period_s = period_s # Starting time self.start_time = time.time() # Mouse event handler for get_hsv def on_mouse(self, event, x, y, flag, param): if event == cv2.EVENT_LBUTTONDBLCLK: # Circle to indicate hsv location, and update frame cv2.circle(self.img_debug, (x, y), 3, (0, 0, 255)) cv2.imshow('hsv_extractor', self.img_debug) # Print values values = self.hsv_frame[y, x] print('H:', values[0], '\tS:', values[1], '\tV:', values[2]) def get_hsv(self): cv2.namedWindow('hsv_extractor') while True: self.grab_frame() # Bottom ROI cv2.rectangle(self.img_debug, (0, cvsettings.HEIGHT_PADDING_BOTTOM-2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2) # Top ROI cv2.rectangle(self.img_debug, (0, cvsettings.HEIGHT_PADDING_TOP-2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2) # Object cv2.rectangle(self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2) self.hsv_frame = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) # Mouse handler cv2.setMouseCallback('hsv_extractor', self.on_mouse, 0) cv2.imshow('hsv_extractor', self.img_debug) key = cv2.waitKey(0) & 0xFF if key == ord('q'): break self.stop_camera() cv2.destroyAllWindows() # Starts camera (needs to be called before run) def start_camera(self): self.camera_started = True self.vs.start() time.sleep(2.0) # Wait for camera to cool def stop_camera(self): self.camera_started = False self.vs.stop() # Grabs frame from camera def grab_frame(self): # Starts camera if it hasn't been started if not self.camera_started: self.start_camera() self.img = self.vs.read() self.img_debug = self.img.copy() # Normalizes our image def normalize_img(self): # Crop img and convert to hsv self.img_roi_bottom = np.copy(self.img[cvsettings.HEIGHT_PADDING_BOTTOM:int(cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT), :]) self.img_roi_top = np.copy(self.img[cvsettings.HEIGHT_PADDING_TOP:int(cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT), :]) self.img_roi_bottom_hsv = cv2.cvtColor(self.img_roi_bottom, cv2.COLOR_BGR2HSV).copy() self.img_roi_top_hsv = cv2.cvtColor(self.img_roi_top, cv2.COLOR_BGR2HSV).copy() # Get our ROI's shape # Doesn't matter because both of them are the same shape self.roi_height, self.roi_width, self.roi_channels = self.img_roi_bottom.shape # Smooth image and convert to bianry image (threshold) # Filter out colors that are not within the RANGE value def filter_smooth_thres(self, RANGE, color): for (lower, upper) in RANGE: lower = np.array(lower, dtype='uint8') upper = np.array(upper, dtype='uint8') mask_bottom = cv2.inRange(self.img_roi_bottom_hsv, lower, upper) mask_top = cv2.inRange(self.img_roi_top_hsv, lower, upper) blurred_bottom = cv2.medianBlur(mask_bottom, 5) blurred_top = cv2.medianBlur(mask_top, 5) # Morphological transformation kernel = np.ones((2, 2), np.uint8) smoothen_bottom = blurred_bottom #cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5) smoothen_top = blurred_top # cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5) """ if self.debug: cv2.imshow('mask bottom ' + color, mask_bottom) cv2.imshow('blurred bottom' + color, blurred_bottom) cv2.imshow('mask top ' + color, mask_top) cv2.imshow('blurred top' + color, blurred_top) """ return smoothen_bottom, smoothen_top # Gets metadata from our contours def get_contour_metadata(self): # Metadata (x,y,w,h)for our ROI contour_metadata = {} for cur_img in [self.thres_yellow_bottom, self.thres_yellow_top, self.thres_blue_bottom, self.thres_blue_top]: key = '' # Blue is left lane, Yellow is right lane if cur_img is self.thres_yellow_bottom: key = 'right_bottom' elif cur_img is self.thres_yellow_top: key = 'right_top' elif cur_img is self.thres_blue_bottom: key = 'left_bottom' elif cur_img is self.thres_blue_top: key = 'left_top' _, contours, hierarchy = cv2.findContours(cur_img.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cur_height, cur_width = cur_img.shape # Get index of largest contour try: areas = [cv2.contourArea(c) for c in contours] max_index = np.argmax(areas) cnt = contours[max_index] # Metadata of contour x, y, w, h = cv2.boundingRect(cnt) # Normalize it to the original picture x += int(cvsettings.WIDTH_PADDING + w / 2) if 'top' in key: y += int(cvsettings.HEIGHT_PADDING_TOP +h /2) else: y += int(cvsettings.HEIGHT_PADDING_BOTTOM + h / 2) contour_metadata[key] = (x, y) self.detected_lane = True # If it throws an error then it doesn't have a ROI # Means we're too far off to the left or right except: # Blue is left lane, Yellow is right lane x = int(cvsettings.WIDTH_PADDING) - cvsettings.CAMERA_WIDTH if 'bottom' in key: y = int(cur_height / 2) + int(cvsettings.HEIGHT_PADDING_BOTTOM + cur_height / 2) else: y = int(cur_height / 2) + int(cvsettings.HEIGHT_PADDING_TOP + cur_height / 2) if 'right' in key: x = int(cur_width)*2 contour_metadata[key] = (x, y) self.detected_lane = False return contour_metadata # Gets the centered coord of the detected lines def get_centered_coord(self): bottom_centered_coord = None top_centered_coord = None left_xy_bottom = self.contour_metadata['left_bottom'] right_xy_bottom = self.contour_metadata['right_bottom'] left_xy_top = self.contour_metadata['left_top'] right_xy_top = self.contour_metadata['right_top'] bottom_xy = (left_xy_bottom[0] + right_xy_bottom[0], left_xy_bottom[1] + right_xy_bottom[1]) bottom_centered_coord = (int(bottom_xy[0] / 2), int(bottom_xy[1] / 2)) top_xy = (left_xy_top[0] + right_xy_top[0], left_xy_top[1] + right_xy_top[1]) top_centered_coord = (int(top_xy[0] / 2), int(top_xy[1] / 2)) # Left can't be greater than right and vice-versa if left_xy_top > right_xy_top: top_centered_coord = (0, top_centered_coord[1]) elif right_xy_top < left_xy_top: top_centered_corrd = (cvsettings.CAMERA_WIDTH, top_centered_coord[1]) if left_xy_bottom > right_xy_bottom: bottom_centered_coord = (0, bottom_centered_coord[1]) elif right_xy_bottom < left_xy_bottom: bottom_centered_coord = (cvsettings.CAMERA_WIDTH, top_centered_coord[1]) return bottom_centered_coord, top_centered_coord # Gets the error of the centered coordinates (x) # Also normlizes their values def get_errors(self): top_error = (float(self.center_coord_top[0]) - float(self.center))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) bottom_error = (float(self.center_coord_bottom[0]) - float(self.center))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) relative_error = (float(self.center_coord_top[0]) - float(self.center_coord_bottom[0]))/float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) return (top_error + relative_error + bottom_error)/3.0 # Object avoidance def where_object_be(self): # We only want to detect objects in our path: center of top region and bottom region # So to save processing speed, we'll only threshold from center of top region to center of bottom region left_x = cvsettings.WIDTH_PADDING right_x = cvsettings.CAMERA_WIDTH # Image region with objects img_roi_object = self.img[cvsettings.OBJECT_HEIGHT_PADDING : int(cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), left_x:right_x] img_roi_object_hsv = cv2.cvtColor(img_roi_object, cv2.COLOR_BGR2HSV).copy() # Filtering color and blurring for (lower, upper) in cvsettings.OBJECT_HSV_RANGE: lower = np.array(lower, dtype='uint8') upper = np.array(upper, dtype='uint8') mask_object = cv2.inRange(img_roi_object_hsv, lower, upper) blurred_object = cv2.medianBlur(mask_object, 25) # Finding position of object (if its there) _, contours, hierarchy = cv2.findContours(blurred_object.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) left_x = self.contour_metadata['left_top'][0] right_x = self.contour_metadata['right_top'][0] for c in contours: areas = cv2.contourArea(c) # Object needs to be larger than a certain area if areas > cvsettings.OBJECT_AREA: x, y, w, h = cv2.boundingRect(c) y += int(cvsettings.OBJECT_HEIGHT_PADDING + h / 2) # Confusing part - this finds the object and makes it think that # it is also a line to avoid bumping into it # It +w and -w to find which line its closest to and then set # the opposite as to be the new left/right lane # e.g. line is closest to left lane (x-w), so left lane new x is # (x+w) distance_to_left = abs(x - left_x) distance_to_right = abs(x+w - right_x) # Make object's left most area the middle of right lane if distance_to_left > distance_to_right: self.contour_metadata['right_top'] = (x, self.contour_metadata['right_top'][1]) # Make object's right most area the middle of left lane elif distance_to_right > distance_to_right: self.contour_metadata['left_top'] = (x+w, self.contour_metadata['left_top'][1]) if self.debug: cv2.circle(self.img_debug, (x+(w/2), y+(h/2)), 5, (240, 32, 160), 2) if self.debug: cv2.imshow('Blurred object', blurred_object) # Where are we relative to our lane def where_lane_be(self): # Running once every period_ms while float(time.time() - self.start_time) < self.period_s: pass # Update time instance self.start_time = time.time() # Camera grab frame and normalize it self.grab_frame() self.normalize_img() # Filter out them colors self.thres_blue_bottom, self.thres_blue_top = self.filter_smooth_thres(cvsettings.BLUE_HSV_RANGE, 'blue') self.thres_yellow_bottom, self.thres_yellow_top = self.filter_smooth_thres(cvsettings.YELLOW_HSV_RANGE, 'yellow') # Get contour meta data self.contour_metadata = self.get_contour_metadata() # Finds objects and (and corrects lane position) # this overwrite contour_metadata #self.where_object_be() # Find the center of the lanes (bottom and top) [we wanna be in between] self.center_coord_bottom, self.center_coord_top = self.get_centered_coord() # Gets relative error between top center and bottom center self.relative_error = self.get_errors() if self.debug: # Drawing locations blue_top_xy = self.contour_metadata['left_top'] blue_bottom_xy = self.contour_metadata['left_bottom'] yellow_top_xy = self.contour_metadata['right_top'] yellow_bottom_xy = self.contour_metadata['right_bottom'] # Circles to indicate lanes cv2.circle(self.img_debug, blue_top_xy, 5, (255, 0, 0), 2) cv2.circle(self.img_debug, blue_bottom_xy, 5, (255, 0, 0), 2) cv2.circle(self.img_debug, yellow_top_xy, 5, (0, 255, 255), 2) cv2.circle(self.img_debug, yellow_bottom_xy, 5, (0, 255, 255), 2) cv2.circle(self.img_debug, self.center_coord_bottom, 5, (0, 255, 0), 3) cv2.circle(self.img_debug, self.center_coord_top, 5, (0, 255, 0), 3) # ROI for object detection cv2.rectangle(self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2) # Displaying image cv2.imshow('img', self.img_debug) #cv2.imshow('img_roi top', self.img_roi_top) #cv2.imshow('img_roi bottom', self.img_roi_bottom) #cv2.imshow('img_hsv', self.img_roi_hsv) cv2.imshow('thres_blue_bottom', self.thres_blue_bottom) cv2.imshow('thres_blue_top', self.thres_blue_top) cv2.imshow('thres_yellow_bottom', self.thres_yellow_bottom) cv2.imshow('thres_yellow_top', self.thres_yellow_top) key = cv2.waitKey(0) & 0xFF # Change 1 to 0 to pause between frames # Use this to calculate fps def calculate_fps(self, frames_no=100): fps = FPS().start() # Don't wanna display window if self.debug: self.debug = not self.debug for i in range(0, frames_no): self.where_lane_be() fps.update() fps.stop() # Don't wanna display window if not self.debug: self.debug = not self.debug print('Time taken: {:.2f}'.format(fps.elapsed())) print('~ FPS : {:.2f}'.format(fps.fps())) # Use this to save images to a location def save_images(self, dirname='dump'): import os img_no = 1 # Makes the directory if not os.path.exists('./' + dirname): os.mkdir(dirname) while True: self.grab_frame() if self.debug: cv2.imshow('frame', self.img) k = cv2.waitKey(1) & 0xFF if k == ord('s'): cv2.imwrite(os.path.join(dirname, 'dump_' + str(img_no) + '.jpg'), self.img) img_no += 1 elif k == ord('q'): break cv2.destroyAllWindows() # Destructor def __del__(self): self.vs.stop() cv2.destroyAllWindows()
class App: def __init__(self, logger, src, ROOT_DIR): self.vs = WebcamVideoStream(src) self.fps = FPS() self.logger = logger self.ROOT_DIR = ROOT_DIR cv2.namedWindow("Webcam") cv2.namedWindow("roi") cv2.namedWindow("stacked") cv2.createTrackbar('dilate kernel', 'roi', 3, 5, self.none) cv2.createTrackbar('erode kernel', 'roi', 2, 5, self.none) cv2.createTrackbar('blackhat kernel', 'roi', 21, 30, self.none) self.mouse = Mouse(window="Webcam") self.gt = Graphics() # self.hist = Hist() self.msg = "draw a rectangle to continue ..." self.font_20 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 20) self.font_10 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 15) self.font_30 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Light.ttf', 30) self.font_40 = ImageFont.truetype(f'{self.ROOT_DIR}/fonts/raleway/Raleway-Medium.ttf', 50) # self.stabilizer = VidStab() self.predictor = parser_v3.Predictor(model_path=f'{self.ROOT_DIR}/models/model_0.1v7.h5', root_dir=self.ROOT_DIR) def run(self): self.vs.start() self.fps.start() self.logger.info("app started ...") frame = self.vs.read() wh, ww, _ = frame.shape cv2.moveWindow("Webcam", 0, 0) cv2.moveWindow("roi", ww, 0) cv2.moveWindow("stacked", 0, wh + 79) self.mouse.rect = ((200, 200), (ww - 200, wh - 200)) while True: frame = self.vs.read() # frame = frame[:, ::-1] # flip orig = frame.copy() # stabilized_frame = self.stabilizer.stabilize_frame( # input_frame=frame, smoothing_window=30, border_size=50) if self.mouse.rect: p1, p2 = self.mouse.rect roi = self.gt.draw_roi(orig, p1, p2) if roi.size != 0: gray_roi = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) dk = cv2.getTrackbarPos("dilate kernel", "roi") ek = cv2.getTrackbarPos("erode kernel", "roi") bk = cv2.getTrackbarPos("blackhat kernel", "roi") thresh = im.thresify(gray_roi, dk, ek, bk) # print(thresh.shape) # overlap thresh on original image mod_thres = cv2.bitwise_not(thresh) #orig[p1[1]:p2[1], p1[0]:p2[0]] = cv2.merge((mod_thres, mod_thres, mod_thres)) # boxes, digits, cnts = component.get_symbols( # thresh, im=roi, draw_contours=True) digits, boxes = component.connect_cnts2(thresh, roi) stacked_digits, resized_digits = component.stack_digits(digits, im.resize) res, latex_image, labels = self.predictor.parse(resized_digits, boxes) # annotate symbols # for label, (pre_label, x_min, y_min, x_max, y_max) in zip(labels, boxes): # self.gt.write(orig, label, (x_min, y_min), self.font_20) # res = self.predictor.parse(resized_digits, boxes) #print(f"res: {res}") self.gt.write(orig, res, (10, wh - wh / 8), self.font_10) # self.gt.draw_boxes(orig, boxes, p1, p2) # self.hist.draw(gray_roi) cv2.imshow('roi', thresh) cv2.imshow('stacked', stacked_digits) # cv2.imshow('latex_image', latex_image) # self.gt.write(orig, self.msg, (10, 10), self.font_20) else: orig[:, :] = cv2.blur(orig, (100, 100)) windowRect = cv2.getWindowImageRect('Webcam') wx, wy, ww, wh = windowRect self.gt.write(orig, self.msg, (100, wh / 2 - 30), self.font_30) cv2.imshow('Webcam', orig) # cv2.imshow('stab', stabilized_frame) self.fps.update() key = cv2.waitKey(1) if (key & 0xFF == ord('q')) or (key & 0xFF == 27): self.logger.info('exiting ...') break elif key & 0xFF == ord('c'): CAPTURED_DIR = f'{self.ROOT_DIR}/screenshots' imageName = f'{CAPTURED_DIR}/{str(time.strftime("%Y_%m_%d_%H_%M"))}.png' cv2.imwrite(imageName, orig) self.logger.info(f'screenshot saved at {imageName}') self.fps.stop() self.vs.stop() cv2.destroyAllWindows() self.logger.info("elapsed time: {:.2f}".format(self.fps.elapsed())) self.logger.info("approx. FPS: {:.2f}".format(self.fps.fps())) def none(self, x): pass
def demo(args): n_models = len(models) model_idx = 0 # content_image = utils.load_image(args.content_image, scale=args.content_scale) content_transform = transforms.Compose([ transforms.ToTensor(), transforms.Lambda(lambda x: x.mul(255)) ]) style_model = TransformerNet() load_model(style_model, args, model_idx) fps = FPS().start() stream = WebcamVideoStream(src=0) # default camera stream.start() # default camera time.sleep(1.0) last_time = time.time() # start fps timer # loop over frames from the video file stream while True: # Change style if time.time() - last_time > SECONDS_PER_MODEL: model_idx = (model_idx + 1) % n_models load_model(style_model, args, model_idx) last_time = time.time() # grab next frame content_image = stream.read() content_image = content_transform(content_image) content_image = content_image.unsqueeze(0) if args.cuda: content_image = content_image.cuda() content_image = Variable(content_image, volatile=True) # update FPS counter fps.update() output = style_model(content_image) if args.cuda: output = output.cpu() output_data = output.data[0].numpy() output_data = np.clip(output_data, 0, 255).astype(np.uint8) output_data = np.transpose(output_data, (1, 2, 0)) key = cv2.waitKey(1) & 0xFF # keybindings for display if key == ord('p'): # pause while True: key2 = cv2.waitKey(1) & 0xff if key2 == ord('p'): # resume break cv2.imshow('Output', output_data) if key in EXIT_KEYS: # exit break elif key in [LEFT_KEY, RIGHT_KEY]: sign = -1 if key == LEFT_KEY else 1 model_idx += sign if model_idx >= n_models: model_idx = 0 elif model_idx < 0: model_idx = n_models - 1 load_model(style_model, args, model_idx) last_time = time.time() stream.stop() stream.stream.release() # stop the timer and display FPS information fps.stop() print("[INFO] elasped time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
if len(sys.argv) - 1 > 0: CAMERA_ID = sys.argv[1] CAM_WIDTH = 960 CAM_HEIGHT = 540 THRESHOLD = 40 PORT = 8000 DEBUG = False cached_image = BytesIO() last_data = [] capture = WebcamVideoStream(src=0) capture.stream.set(cv2.CAP_PROP_FRAME_WIDTH, CAM_WIDTH) capture.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, CAM_HEIGHT) time.sleep(2) capture.start() time.sleep(2) def get_host_ip(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) ip = s.getsockname()[0] s.close() return ip def init_claim(): claims = [ {"type": "retract", "fact": [["id", get_my_id_str()], ["postfix", ""]]}, {"type": "claim", "fact": [ ["id", get_my_id_str()], ["id", "0"],
def main(): # ######################### # # ARGUMENT DEFINITIONS # # ######################### # # ARGUMENT DEFINITION # Construct the argument parser and parse the arguments. # This is where the input arguments for this program is read in. # These arguments are read in through the Python command line. Otherwise, # if they are not specified, they will be the default values. ap = argparse.ArgumentParser() # Number of seconds to record the video for. # Keep -1 to keep recording indefinitely until user input terminates program. # -1 will put it in TTL mode, where it will wait for a pulse before starting. # Otherwise it will just start immediately and record for the provided amount of seconds. # Specifying a duration that isn't -1 is best used for benchmarking. # Default: -1 ap.add_argument("-s", "--seconds", type=int, default=-1, help="Number of seconds to record video for (default: -1)") # FPS of the output video. # Avoid values that are too high. Otherwise the output video will # not be synchronized and will be too short relative to the recording time. # Default: 5 ap.add_argument("-f", "--fps", type=int, default=5, help="Output video file's FPS rate (default: 5)") # Codec used for the output video. # Testing between different codecs indicate that we should consider between # YUV (codec: IYUV) or XVID (codec: XVID). YUV will give better FPS performance # on this computer, but will cause output files to be huge. # XVID is fairly slower, but files are relatively small and compressed. # Potential alternatives can also include MJPG, which is the fastest, but highly compressed # and will affect video quality more so than the other codecs. ap.add_argument( "-c", "--codec", type=str, default='MJPG', help= "Codec to use; use IYUV for better FPS, XVID for more compression, MJPG for fastest FPS but hurts quality (default: \"MJPG\")" ) # Path to save the recorded video files. # Default: outputX.avi, where X is the respective camera the video was recording from. ap.add_argument( "-o1", "--out1", type=str, default='output1.avi', help= "Name of the output file path for camera 1 (default: \"output1.avi\")") ap.add_argument( "-o2", "--out2", type=str, default='output2.avi', help= "Name of the output file path for camera 2 (default: \"output2.avi\")") ap.add_argument( "-o3", "--out3", type=str, default='output3.avi', help= "Name of the output file path for camera 3 (default: \"output3.avi\")") ap.add_argument( "-o4", "--out4", type=str, default='output4.avi', help= "Name of the output file path for camera 4 (default: \"output4.avi\")") # Camera indexes. This is the USB index the program will connect to in order to # read from the cameras. # Default: 0, 1, 2, 3 for cameras 1, 2, 3, and 4 respectively. # Highly RECOMMENDED to find out and specify the correct indices for the cameras. # There is a high probability that the default values are incorrect for most cases. ap.add_argument("-c1", "--cam1", type=int, default=0, help="Index value for camera 1 (default: 0)") ap.add_argument("-c2", "--cam2", type=int, default=1, help="Index value for camera 2 (default: 1)") ap.add_argument("-c3", "--cam3", type=int, default=2, help="Index value for camera 3 (default: 2)") ap.add_argument("-c4", "--cam4", type=int, default=3, help="Index value for camera 4 (default: 3)") # Conclude parsing the arguments. args = vars(ap.parse_args()) # ##################### # # PRINT INFO # # ##################### # # PRELIMINARY INFORMATION # Print out the parameter information before starting the stream. print('[INST] Starting 4K camera program.') print( '[INST] Make sure start.py on the rPi is already initiated and ready to start!\n' ) print('[STAT] Resolution: ' + str(WIDTH) + 'x' + str(HEIGHT)) print('[STAT] camera 1 index: ' + str(args["cam1"])) print('[STAT] camera 2 index: ' + str(args["cam2"])) print('[STAT] camera 3 index: ' + str(args["cam3"])) print('[STAT] camera 4 index: ' + str(args["cam4"])) print('[STAT] seconds: ' + str(args["seconds"])) print('[STAT] fps: ' + str(args["fps"])) print('[STAT] codec: ' + str(args["codec"])) print('[STAT] output path 1: ' + str(args["out1"])) print('[STAT] output path 2: ' + str(args["out2"])) print('[STAT] output path 3: ' + str(args["out3"])) print('[STAT] output path 4: ' + str(args["out4"])) print('[STAT] TTL Train data save path: ' + cwd) # ##################### # # INITIALIZE CAMERAS # # ##################### # # You can manually edit the codec here if needed. FOURCC_CODEC = cv2.VideoWriter_fourcc(*args["codec"]) # INITIALIZE CAMERA STREAMS # Creates a *threaded* video stream, allow the camera sensor to warm-up, # and starts the FPS counter # Camera 1 setup print("\n[INFO] Initializing video camera stream 1...") vs1 = WebcamVideoStream(src=args["cam1"]) # Record from index vs1.fps = args["fps"] # Set FPS vs1.num_frames = args[ "seconds"] * vs1.fps # Set total amount of frames to record vs1.out = cv2.VideoWriter(args["out1"], FOURCC_CODEC, vs1.fps, vs1.resolution) vs1.frame_index = 1 / vs1.fps # Camera 2 setup print("\n[INFO] Initializing video camera stream 2...") vs2 = WebcamVideoStream(src=args["cam2"]) # Record from index vs2.fps = args["fps"] # Set FPS vs2.num_frames = args[ "seconds"] * vs2.fps # Set total amount of frames to record vs2.out = cv2.VideoWriter(args["out2"], FOURCC_CODEC, vs2.fps, vs2.resolution) vs2.frame_index = 1 / vs2.fps # Camera 3 setup print("\n[INFO] Initializing video camera stream 3...") vs3 = WebcamVideoStream(src=args["cam3"]) # Record from index vs3.fps = args["fps"] # Set FPS vs3.num_frames = args[ "seconds"] * vs3.fps # Set total amount of frames to record vs3.out = cv2.VideoWriter(args["out3"], FOURCC_CODEC, vs3.fps, vs3.resolution) vs3.frame_index = 1 / vs3.fps # Camera 4 setup print("\n[INFO] Initializing video camera stream 4...") vs4 = WebcamVideoStream(src=args["cam4"]) # Record from index vs4.fps = args["fps"] # Set FPS vs4.num_frames = args[ "seconds"] * vs4.fps # Set total amount of frames to record vs4.out = cv2.VideoWriter(args["out4"], FOURCC_CODEC, vs4.fps, vs4.resolution) vs4.frame_index = 1 / vs4.fps # ################# # # READ ON PULSES # # ################# # # INITIALIZE TASK for TTL # At this point we keep reading for pulses from the ON port until we get a signal to actually start. if args["seconds"] == -1: # If seconds is set to -1, it will be considered in TTL mode and will wait for a pulse before starting. # Set up serial port to read start pulse COM_PORT = 'COM4' # USB Serial COM Port ser = serial.Serial(port=COM_PORT, baudrate=115200, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE, bytesize=serial.EIGHTBITS, timeout=1) # Wait for pulse before starting print('[INFO] Waiting for TTL pulse...') PULSE_START = False # Keep reading from the port until a valid pulse is detected. while PULSE_START is False: # Read input data pulse = ser.read(1) # Check if ON pulse occurred, if so, start recording. if pulse is b'\x00': PULSE_START = True TIMESTAMPS.append([str(datetime.datetime.now()), pulse]) print('\n[INFO] START PULSE DETECTED') # ######################### # # START VIDEO STREAMS # # ######################### # # Start the video streams vs1.start() vs2.start() vs3.start() vs4.start() print("\n[INFO] Recording...") # ######################### # # WAIT FOR END CONDITION # # ######################### # # INITIALIZE IF RECORDING BY TIME # This only runs if a duration for seconds is provided. if args["seconds"] != -1: # Keep recording until the time duration is reached. print('[INFO] Initialized time-based recording...') start = datetime.datetime.now() end = datetime.datetime.now() while (end - start).total_seconds() < args["seconds"]: # Do nothing, wait out the time period end = datetime.datetime.now() # INITIALIZE IF RECORDING WITH TTL # STOP NIDAQ TASK for TTL # Here we keep recording from the off port, waiting for a signal to stop. # The cameras will keep recording until a valid off pulse is read. if args["seconds"] == -1: print('[INFO] Initialized TTL-based recording...') # Keep reading in TTL pulse to know when to stop PULSE_STOP = False # Keep recording until stop pulse is sent while PULSE_STOP is False: # Read from serial pulse = ser.read(1) # Add to timestamp if pulse is b'\x00': TIMESTAMPS.append([str(datetime.datetime.now()), pulse]) # Check if OFF pulse occurred, and if so, stop recording. if pulse is b'': PULSE_STOP = True print('\n[INFO] STOP PULSE DETECTED\n') # ################# # # STOP RECORDING # # ################# # # Stop recording from all the streams. # All these streams must be stopped via multithreading due to the five second sleep command # in the stop function. Otherwise there would be a five second gap between each video file closing. # The five second gap is required to prevent crashes from premature closing. print('[INFO] Recording stopping...') # Create the threads vs1_thread = Thread(target=vs1.stop, args=()) vs2_thread = Thread(target=vs2.stop, args=()) vs3_thread = Thread(target=vs3.stop, args=()) vs4_thread = Thread(target=vs4.stop, args=()) # Start all the threads vs1_thread.start() vs2_thread.start() vs3_thread.start() vs4_thread.start() # Wait for all of the threads to finish vs1_thread.join() vs2_thread.join() vs3_thread.join() vs4_thread.join() print('[INFO] All recordings stopped!') # ######################### # # PRINT FPS INFORMATION # # ######################### # # Print out FPS here. # It is CRUCIAL to note that the average FPS for each camera must be extremely close to the target # FPS, otherwise there will be timing and synchronization issues (Videos will be shorter than # the expected length!) average_fps1 = vs1.FPS_counter.fps() print('\nAverage FPS for camera 1: \t' + str(average_fps1)) print('Target FPS: \t' + str(vs1.fps)) if round(average_fps1) < vs1.fps: print( '[WARNING]: Average output FPS for camera 1 does not closely match specified FPS!' ) average_fps2 = vs2.FPS_counter.fps() print('\nAverage FPS for camera 2: \t' + str(average_fps2)) print('Target FPS: \t' + str(vs2.fps)) if round(average_fps2) < vs2.fps: print( '[WARNING]: Average output FPS for camera 2 does not closely match specified FPS!' ) average_fps3 = vs3.FPS_counter.fps() print('\nAverage FPS for camera 3: \t' + str(average_fps3)) print('Target FPS: \t' + str(vs3.fps)) if round(average_fps3) < vs3.fps: print( '[WARNING]: Average output FPS for camera 3 does not closely match specified FPS!' ) average_fps4 = vs4.FPS_counter.fps() print('\nAverage FPS for camera 4: \t' + str(average_fps4)) print('Target FPS: \t' + str(vs4.fps)) if round(average_fps4) < vs4.fps: print( '[WARNING]: Average output FPS for camera 4 does not closely match specified FPS!' ) print('\n[INFO] Saving TTL pulse data to ' + str(cwd)) # Save data to csv file in train_recordings folder with open(cwd, 'w') as csvfile: writer = csv.writer(csvfile, delimiter=',') writer.writerow(['Timestamp', 'Value']) for i in range(len(TIMESTAMPS)): writer.writerow([TIMESTAMPS[i][0], TIMESTAMPS[i][1]]) csvfile.close() print('\nPROGRAM CONCLUDED!') return
class OpenCVController: # PyGame constants (RGB value for white & black and the radius of the eyeballs to draw) RGB_WHITE = (255, 255, 255) RGB_BLACK = (0, 0, 0) EYEBALL_RADIUS = 30 # Font for text on webcam display (put here to simply our writing commands elsewhere and enable uniformity) font = cv2.FONT_HERSHEY_SIMPLEX # Incremented after each photo is taken - this allows each photo to have a unique name and not be overwritten imageNumber = 0 # Define lower & upper boundaries in HSV color space (for object following/tracking) greenLower = np.array([48, 100, 50]) greenUpper = np.array([85, 255, 255]) # Constructor/initalizer for this class def __init__(self): # Connect to the Arduino via serial port connection. If we're testing without the Arduino then set it to # False in the line below so we don't get stuck waiting/hanging for the serial connection to complete self.serialPort = self.com_connect() if True else None # True for running - False for testing # Variables to hold last command sent to Arduino and when it was sent (epoch seconds). Note: # lastCommandSentViaSerialTime ended up not being utilized - but its purpose was to send duplicate commands # after some certain amount of time in case the Arduino needed it for some reason (it did not with our # current design) self.lastCommandSentViaSerial = None self.lastCommandSentViaSerialTime = None # Connect to webcam video source - WebcamVideoStream is a threaded class. By including it here, # all our functions will be able to use it (since only one is run at a time) and we won't have to restart our # video streaming each time we want to run a different function/capability. self.WebcamVideoStreamObject = WebcamVideoStream(src=1) # Save the original exposure and gain of the webcam for restoring later... (we will modify these parameters # when we notice that the motion of the robot is impairing the OpenCV/image-processing capabilities (motion # blur makes it hard to detect april tags, so by changing these parameters we can capture/detect these tags # even while the robot is moving quite fast). We'd like to restore the camera to its original settings when # it's not needed to specialize/adjust them because that enables the camera to choose the best settings for # the environment and change its exposure automatically. self.originalExposure = self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_EXPOSURE) self.originalGain = self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_GAIN) # Note: original exposure is probably like 0.015 or something small (depending on the lightning) and adjusts # automatically # Start the webcam streaming operation and set the object attribute to use... self.vs = self.WebcamVideoStreamObject.start() # Allow camera to warm up... time.sleep(2.0) # subprocess.check_call("v4l2-ctl -d /dev/video1 -c exposure_absolute=40",shell=True) # print("New Gain") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_GAIN))) # print("New exposure") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_EXPOSURE))) # print("Frame width") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_FRAME_WIDTH))) # print("Frame height") print(str(self.WebcamVideoStreamObject.stream.get(cv2.CAP_PROP_FRAME_HEIGHT))) # Tries to open a specific URL to confirm that we are connected to the internet - this is never used in the # codebase because some of our startup steps for our code already depended on internet connection. @staticmethod def internet_on(): try: urlopen('http://216.58.192.142', timeout=1) return True except urllib2.URLError as err: return False # Connected to Arduino for serial communication. This method will hang/wait until it can make the connection, # so make sure not to call this method if you aren't testing with an Arduino connected (see __init__ above) @staticmethod def com_connect(): ser = None connection_made = False while not connection_made: if os.path.exists('/dev/ttyUSB0'): connection_made = True ser = serial.Serial('/dev/ttyUSB0', 9600, write_timeout=0) print("Connected to Serial") if os.path.exists('/dev/ttyACM1'): connection_made = True ser = serial.Serial('/dev/ttyACM1', 115200, write_timeout=0) return ser # Send serial command to Arduino. If the last command that we've sent is the same as the one we're trying to send # now, then ignore it since the Arduino already has the up-to-date command. Note that there's a commented out # section of this code that made it so that even if the command was a duplicate of the last send one, # it would still send the command as long as a certain time period had passed. Depending on how the Arduino code # worked this may have been necessary, but we did not need it. def send_serial_command(self, direction_enum, dataToSend): # If this command is different than the last command sent, then we should sent it # Or if it's the same command but it's been 1 second since we last sent a command, then we should send it if self.serialPort is not None: if self.lastCommandSentViaSerial != direction_enum: self.serialPort.write(dataToSend) self.lastCommandSentViaSerial = direction_enum self.lastCommandSentViaSerialTime = time.time() # elif (time.time() - self.lastCommandSentViaSerialTime > 1): # TODO also need null check here # self.serialPort.write(dataToSend) # self.lastCommandSentViaSerialTime = time.time() else: pass # Do nothing - same command sent recently # Call this when closing this openCV process. It will stop the WebcamVideoStream thread, close all openCV # windows, and close the SerialPort as long as it exists (if we're connected to an Arduino). def cleanup_resources(self): # TODO change the order of stream release and stop() - see what happens self.vs.stop() # self.vs.stream.release() # TODO this should be called but is throwing errors - it works as-is though cv2.destroyAllWindows() # Not necessary since I do it at the end of every method if self.serialPort is not None: # Close serialPort if it exists self.serialPort.close() # Send an email with optional attachments specified (see usage in this file) @staticmethod def send_mail(send_from: str, subject: str, text: str, send_to: list, files=None): username = '******' password = '******' msg = MIMEMultipart() msg['From'] = send_from msg['To'] = ', '.join(send_to) msg['Subject'] = subject msg.attach(MIMEText(text)) for f in files or []: with open(f, "rb") as fil: ext = f.split('.')[-1:] attachedfile = MIMEApplication(fil.read(), _subtype=ext) attachedfile.add_header( 'content-disposition', 'attachment', filename=basename(f)) msg.attach(attachedfile) smtp = smtplib.SMTP(host="imap.gmail.com", port=587) smtp.starttls() smtp.login(username, password) smtp.sendmail(send_from, send_to, msg.as_string()) smtp.close() # Countdown timer and then take a photo using a frame from the WebcamVideoStream. The photo is sent to the emails # of the group members. Note that the quality of the image can likely be increased by tuning the webcam settings # (I did not because it was not necessary). def take_photo(self, cvQueue: Queue): # Creating a window for later use cv2.namedWindow('result') cv2.resizeWindow('result', 600, 600) # Use the below stuff to do the countdown timer startTime = time.time() waitTimeSec = 5 strSec = "54321" while True: # Grab frame - break if we don't get it (some unknown error occurred) frame = self.vs.read() if frame is None: print("ERROR - frame read a NONE") break frame = imutils.resize(frame, width=600) currTime = time.time() timeDiff = int(currTime - startTime) # seconds if timeDiff < waitTimeSec: # Not time to take the photo yet # Put time countdown on the frame cv2.putText(frame, strSec[timeDiff], (250, 300), self.font, 5, (200, 255, 155), 4, cv2.LINE_AA) cv2.imshow("result", frame) else: # Time's up - take the photo and do post-processing cv2.imshow("result", frame) time.sleep(1) # Freeze the screen so that it's obvious that the photo was taken # Save the image in the current working directory cv2.imwrite(filename='saved_img_' + str(self.imageNumber) + '.jpg', img=frame) cv2.destroyAllWindows() # Close the CV windows # Send email username = '******' password = '******' sendList = ['*****@*****.**', '*****@*****.**', '*****@*****.**'] filesToSend = [os.path.abspath('saved_img_' + str(self.imageNumber) + '.jpg')] # if self.internet_on(): self.send_mail(send_from=username, subject="test", text="text", send_to=sendList, files=filesToSend) # else: # print("Internet is not on - did not send email") self.imageNumber += 1 # Increment for next possible photo break # Close application on 'q' key press or if new stuff added to queue key = cv2.waitKey(1) & 0xFF if (key == ord("q")) or (not cvQueue.empty()): # We've been requested to leave ... # Don't destroy everything - just destroy cv2 windows ... webcam still runs cv2.destroyAllWindows() break # Utilizes the person/eyeball_follow algorithms to move to the requested april tag and reach a desired distance # from the tag. This function is called multiple times (once for each april tag we want to go to). The variables # isFirstUse and isLastUse are used to avoid setting and resetting the webcam's gain and exposure rapidly back # and forth (this creates a problem). By using these variables we set it only on the first april tag and reset it # back to the original specifications on the last tag. def april_following(self, desiredTag, desiredDistance, cvQueue: Queue, isFirstUse, isLastUse): # Fast-fail. If there is something on the cvQueue then that means we need to respond to it. There are # multiple calls of april_following(...) being made in succession in a for-loop to get the robot to a # destiantion. We want to quickly exit from each of the calls in this situation. if not cvQueue.empty(): return # Tune the webcam to better see april tags while robot is moving # (compensating for motion blur). Restore settings when done. # These settings can be played with to create the best effect (along with other settings if you want) if isFirstUse: self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_EXPOSURE, 0.5) self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_GAIN, 1) # Frame is considered to be 600x600 (after resize) (actually it's like 600x400) # Below are variables to set what we consider center and in-range (these numbers are in pixels) radiusInRangeLowerBound, radiusInRangeUpperBound = desiredDistance - 10, desiredDistance + 10 centerRightBound, centerLeftBound = 400, 200 radiusTooCloseLowerLimit = 250 # When turning to search for the desiredTag, we specify time to turn, and time to wait after each semi-turn. # Note that these two variables are NO LONGER USED! By adjusting the exposure to reduce the effects of motion # blur, we no longer have to do this turn-and-stop manuever to search for tags around us. Just rotating works # fine. searchingTimeToTurn = 0.3 # seconds searchingTimeToHalt = 1.0 # seconds # Creating a window for later use cv2.namedWindow('result') cv2.resizeWindow('result', 600, 600) # Variables to 'smarten' the following procedure. See their usage below. objectSeenOnce = False # Object has never been seen before leftOrRightLastSent = None # Keep track of whether we sent left or right last firstTimeObjectNotSeen = None # Holds timestamp (in seconds) of the first time we haven't been able to see # the tag. We don't want to instantly start freaking out and turning around looking for the tag because it's # very possible it was lost in some bad frame, so we wait some X number of seconds before looking around ( # this is what this timestamp is used for). # Initialize apriltag detector options = apriltag.DetectorOptions( families='tag36h11', border=1, nthreads=1, quad_decimate=1.0, quad_blur=0.0, refine_edges=True, refine_decode=True, refine_pose=False, debug=False, quad_contours=True) det = apriltag.Detector(options) # TODO delete this block when done (not necessary to so we kept it - just thought removing extra details # would speed up performance) start = time.time() num_frames = 0 inPosition = False numHalts = 0 while True: # Grab frame - break if we don't get it (some unknown error occurred) frame = self.vs.read() if frame is None: break # TODO delete this block when done (same as above TODO) end = time.time() seconds = end - start num_frames += 1 fps = 0 if (seconds == 0) else num_frames / seconds frame = imutils.resize(frame, width=600) # frame = cv2.filter2D(frame, -1, np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])) # Sharpen image gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Use grayscale image for detection res = det.detect(gray) # Run the image through the apriltag detector and get the results commandString = None # Stores string to print on the screen (the current command to execute) # Check if the desiredTag is visible tagObject = None for r in res: if r.tag_id == desiredTag: tagObject = r if tagObject is None: # We don't see the tag that we're looking for # Don't see the tag? Possibly just bad frame, lets wait 2 seconds and then start turning numHalts += 1 # TODO delete all the numHalt tracking stuff (this was to keep track and lessen the # effects of motion blur ... we kept it since it didn't affect performance). if firstTimeObjectNotSeen is None: firstTimeObjectNotSeen = time.time() self.send_serial_command(Direction.STOP, b'h') commandString = "STOP" else: secondsOfNoTag = time.time() - firstTimeObjectNotSeen if secondsOfNoTag > 2: # Haven't seen our tag for more than 2 seconds if leftOrRightLastSent is not None: if leftOrRightLastSent == Direction.RIGHT: self.send_serial_command(Direction.RIGHT, b'r') commandString = "SEARCHING: GO RIGHT" elif leftOrRightLastSent == Direction.LEFT: self.send_serial_command(Direction.LEFT, b'l') commandString = "SEARCHING: GO LEFT" else: # variable hasn't been set yet (seems unlikely), but default to left self.send_serial_command(Direction.LEFT, b'r') commandString = "DEFAULT SEARCHING: GO RIGHT" # We've sent the command now wait half a second and then send a halt (WE DON"T NEED THIS ANYMORE) # time.sleep(searchingTimeToTurn) # self.send_serial_command(Direction.STOP, b'h'); # time.sleep(searchingTimeToHalt) else: # Keep waiting - 2 seconds haven't elapsed self.send_serial_command(Direction.STOP, b'h') commandString = "STOP" else: # We see the desired tag! # Reset firstTimeObjectNotSeen to None for the next time we can't find the tag if firstTimeObjectNotSeen is not None: firstTimeObjectNotSeen = None # Set objectSeenOnce to True if isn't already if not objectSeenOnce: objectSeenOnce = True # Get the corners and draw a minimally enclosing circle of it # and get the x/y/radius information of that circle to use for our navigation corners = np.array(tagObject.corners, dtype=np.float32).reshape((4, 2, 1)) cornersList = [] for c in corners: cornersList.append([int(x) for x in c]) cornersList = np.array(cornersList, dtype=np.int32) # Turn the list into a numpy array ((x, y), radius) = cv2.minEnclosingCircle(cornersList) M = cv2.moments(cornersList) # Grab the desired information... center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) filteredPtsRadius = [radius] filteredPtsX = [center[0]] filteredPtsY = [center[1]] # Draw circle and center point on the frame cv2.circle(frame, (int(x), int(y)), int(filteredPtsRadius[0]), (0, 255, 255), 2) cv2.circle(frame, center, 5, (0, 0, 255), -1) # Determine what command to send to the Arudino (motors) if filteredPtsRadius[0] > radiusTooCloseLowerLimit: commandString = "MOVE BACKWARD - TOO CLOSE TO TURN" self.send_serial_command(Direction.BACKWARD, b'b') elif filteredPtsX[0] > centerRightBound: commandString = "GO RIGHT" self.send_serial_command(Direction.RIGHT, b'r') if leftOrRightLastSent != Direction.RIGHT: leftOrRightLastSent = Direction.RIGHT elif filteredPtsX[0] < centerLeftBound: commandString = "GO LEFT" self.send_serial_command(Direction.LEFT, b'l') if leftOrRightLastSent != Direction.LEFT: leftOrRightLastSent = Direction.LEFT elif filteredPtsRadius[0] < radiusInRangeLowerBound: commandString = "MOVE FORWARD" self.send_serial_command(Direction.FORWARD, b'f') elif filteredPtsRadius[0] > radiusInRangeUpperBound: commandString = "MOVE BACKWARD" self.send_serial_command(Direction.BACKWARD, b'b') elif radiusInRangeLowerBound < filteredPtsRadius[0] < radiusInRangeUpperBound: commandString = "STOP MOVING - IN RANGE" self.send_serial_command(Direction.STOP, b'h') inPosition = True # Put text on the camera image to display on the screen cv2.putText(frame, 'center coordinate: (' + str(filteredPtsX[0]) + ',' + str(filteredPtsY[0]) + ')', (10, 60), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) cv2.putText(frame, 'filtered radius: (' + str(filteredPtsRadius[0]) + ')', (10, 90), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) # Show FPS and number of halts (this stuff will be on the frame regardless of whether we see our desired # tag or not) (TODO delete this stuff later if we don't want it) cv2.putText(frame, commandString, (10, 30), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) cv2.putText(frame, 'FPS: (' + str(fps) + ')', (10, 120), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) cv2.putText(frame, 'numHalts: (' + str(numHalts) + ')', (10, 150), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) # Display frame cv2.imshow("result", frame) # Close application on 'q' key press, new stuff on queue, or if we've reached our destination key = cv2.waitKey(1) & 0xFF if (key == ord("q")) or (not cvQueue.empty()) or inPosition: self.send_serial_command(Direction.STOP, b'h'); # Restore webcam settings if not inPosition or (inPosition and isLastUse): # Reset the webcam to its original exposure and gain self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_EXPOSURE, self.originalExposure) self.WebcamVideoStreamObject.stream.set(cv2.CAP_PROP_GAIN, self.originalGain) # Activate auto_exposure (which is what the webcam starts out with by default but we mess it up # by changing the exposure manually). subprocess.check_call("v4l2-ctl -d /dev/video1 -c exposure_auto=3", shell=True) cv2.destroyAllWindows() break # Used to priortize the coordinate information provided by closer tags as opposed to the farther aways ones ( # which would likely have less reliable information). @staticmethod def calc_weight(p1, p2): max_weight = 150 dist = np.linalg.norm(p1 - p2) return max_weight / dist # Get an estimate of our current x and z coorindates and return them so they can be passed through to the chat # server and disseminated to the other teams (for the distress signal). def get_coordinates(self, cvQueue: Queue): # To get the coordinate, we rotate on our axis some X number of times to form images that compose a complete # 360 degree view of our surroundings. We use each image (as long as there are april tags in it) to get a (x, # z) coordinate value, and then we choose which (x,z) coordinate to return based off of which we deem the # most correct/reliable (this decision is shown in the code below) # When turning to search for the desiredTag, we specify time to turn, and time to wait after each semi-turn. # We do this because we want a stable photo/shot at each searchingTimeToTurn = 0.5 # seconds searchingTimeToHalt = 2.0 # seconds # Note that refine_pose is set to True (takes more work/processing but hopefully gets better coordinates) options = apriltag.DetectorOptions( families='tag36h11', border=1, nthreads=1, quad_decimate=1.0, quad_blur=0.0, refine_edges=True, refine_decode=True, refine_pose=True, debug=False, quad_contours=True) det = apriltag.Detector(options) # Load camera data with open('cameraParams.json', 'r') as f: data = json.load(f) cameraMatrix = np.array(data['cameraMatrix'], dtype=np.float32) distCoeffs = np.array(data['distCoeffs'], dtype=np.float32) # Load world points world_points = {} with open('worldPoints.json', 'r') as f: data = json.load(f) for k, v in data.items(): world_points[int(k)] = np.array(v, dtype=np.float32).reshape((4, 3, 1)) # Variables for final decision coordinates_list = [] iterationNumber = 1 numIterations = 10 while True: # Rotate camera by going left by some amount self.send_serial_command(Direction.LEFT, b'l') time.sleep(searchingTimeToTurn) self.send_serial_command(Direction.STOP, b'h') time.sleep(searchingTimeToHalt) # Now lets read the frame (while the robot is halted so that image is clean) frame = self.vs.read() if frame is None: print("ERROR - frame read a NONE") break # Use grayscale image for detection gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) res = det.detect(gray) # Check how many tags we see... if it's 0 then ignore this frame and move on to capturing the next frame numTagsSeen = len(res) print("\nNumber of tags seen", numTagsSeen) # TODO remove if numTagsSeen > 0: poses = [] # Store poses from each tag to average them over tagRadiusList = [] # Store tag radius' to determine the largest for r in res: # Iterate over each tag in the frame corners = r.corners tag_id = r.tag_id corners = np.array(corners, dtype=np.float32).reshape((4, 2, 1)) cornersList = [] for c in corners: cornersList.append([int(x) for x in c]) cornersList = np.array(cornersList, dtype=np.int32) # Turn into numpy array (openCV wants this) # Draw circle around tag using its corners & get radius of that tag ((x, y), radius) = cv2.minEnclosingCircle(cornersList) filteredPtsRadius = [radius] # Solve pose ((x,z) coordinates) r, rot, t = cv2.solvePnP(world_points[tag_id], corners, cameraMatrix, distCoeffs) # get rotation and translation vector using solvePnP rot_mat, _ = cv2.Rodrigues(rot) # convert to rotation matrix R = rot_mat.transpose() # Use rotation matrix to get pose = -R * t (matrix mul w/ @) pose = -R @ t weight = self.calc_weight(pose, world_points[tag_id][0]) poses.append((pose, weight)) tagRadiusList.append(filteredPtsRadius) # Done iterating over the tags that're seen in the frame... # Now get the average pose across the tags and get the largest tag radius that we saw. # We will store the (x,z) coordinate that we calculate, and we'll also # store the largest radius for a tag that we've seen in this frame. avgPose = sum([x * y for x, y in poses]) / sum([x[1] for x in poses]) largestTagRadius = max(tagRadiusList) coordinates = (avgPose[0][0], avgPose[2][0], largestTagRadius) print(str(coordinates)) # TODO remove this coordinates_list.append(coordinates) # Display frame cv2.imshow('frame', frame) # If we've completed our numIterations, then choose the coordinate # and return (do closing operations too) if iterationNumber == numIterations: if len(coordinates_list) > 0: # TODO 2 things we can try here ... # 1) The coordinate to return is the one with the smallest z-coordinate # (which essentially means it's closest to those tags that it used) # BUT this value seems to vary a lot and I don't think it's reliable # 2) I have saved the largest radius for a tag seen for each of these # coordinates, so I can use that (which I bet is more reliable) # I will go with approach number 2 # coordinateToReturn = min(coordinates_list, key=lambda x: x[1]) # Approach (1) coordinateToReturn = max(coordinates_list, key=lambda x: x[2]) # Approach (2) coordinateToReturn = ( int(coordinateToReturn[0]), int(coordinateToReturn[1])) # This stays regardless else: coordinateToReturn = (0, -1) # TODO set to some default outside the door # Cleanup and return cv2.destroyAllWindows() print("Value to return:") # TODO remove print(coordinateToReturn) # TODO remove return coordinateToReturn else: # Still have iterations to go, increment the value iterationNumber += 1 # Q to quit key = cv2.waitKey(1) & 0xFF if (key == ord("q")) or (not cvQueue.empty()): self.send_serial_command(Direction.STOP, b'h') cv2.destroyAllWindows() break # Follow a person (represented by a green folder). If the second argument (run_py_eyes) is set to True then we'll # create eyeballs using PyGame that follow the user (folder) as it moves around. def person_following(self, run_py_eyes, cvQueue: Queue): # Frame is considered to be 600x600 (after resize) # Below are variables to set what we consider center and in-range radiusInRangeLowerBound, radiusInRangeUpperBound = 80, 120 centerRightBound, centerLeftBound = 400, 200 radiusTooCloseLowerLimit = 250 # Creating a window for later use cv2.namedWindow('result') cv2.resizeWindow('result', 600, 600) # Variables to 'smarten' the following procedure (see usage below) objectSeenOnce = False # Object has never been seen before leftOrRightLastSent = None # Keep track of whether we sent left or right last # TODO delete this block when done start = time.time() num_frames = 0 # PyEyes Setup... Note that I've done some performance tinkering with pygame. Instead of redrawing the entire # frame on each iteration, I only turn the previously drawn pupils of the last frame white (to match the # background) and draw the new pupils. I also enable some performance enhancements and disable some unneeded # functionality. This kept out frame rate at a reliable level. if run_py_eyes: screen = pygame.display.set_mode((1024, 600), DOUBLEBUF) screen.set_alpha(None) # Not needed, so set it to this for performance improvement surface = pygame.display.get_surface() # Draw the eyeballs (without pupils) and white background that we'll use for the rest of the process screen.fill(self.RGB_WHITE) # Fill PyGame screen (white background) pygame.draw.circle(surface, self.RGB_BLACK, (256, 300), 255, 15) pygame.draw.circle(surface, self.RGB_BLACK, (768, 300), 255, 15) pygame.display.flip() rects = [] while True: # Reset to default pupil coordinates and width (in case no object is found on this iteration) leftx, lefty, width = 256, 350, 0 # Grab frame - break if we don't get it (some unknown error occurred) frame = self.vs.read() if frame is None: print("ERROR - frame read a NONE") break # TODO delete this block when done (if you want) end = time.time() seconds = end - start num_frames += 1 fps = 0 if (seconds == 0) else num_frames / seconds # Resize the frame, blur it, and convert it to the HSV color space frame = imutils.resize(frame, width=600) blurred = cv2.GaussianBlur(frame, (5, 5), 0) hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV) # construct a mask for the desired color, then perform a series of dilations and erosions to # remove any small blobs left in the mask mask = cv2.inRange(hsv, self.greenLower, self.greenUpper) mask = cv2.erode(mask, None, iterations=2) # TODO: these were 3 or 5 before (more small blob removal) mask = cv2.dilate(mask, None, iterations=2) # find contours in the mask and initialize the current (x, y) center of the ball cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) cnts = imutils.grab_contours(cnts) commandString = None # Only proceed if at least one contour was found # If nothing is found, then look around OR send the STOP command to halt movement (depends on situation) if len(cnts) == 0: # If we haven't seen the object before, then we'll stay halted until we see one. If we HAVE seen the # object before, then we'll move in the direction (left or right) that we did most recently if not objectSeenOnce: self.send_serial_command(Direction.STOP, b'h') commandString = "STOP" else: # Object has been seen before if leftOrRightLastSent is not None: if leftOrRightLastSent == Direction.RIGHT: self.send_serial_command(Direction.RIGHT, b'r') commandString = "SEARCHING: GO RIGHT" elif leftOrRightLastSent == Direction.LEFT: self.send_serial_command(Direction.LEFT, b'l') commandString = "SEARCHING: GO LEFT" else: # variable hasn't been set yet (seems unlikely), but default to left self.send_serial_command(Direction.LEFT, b'l') commandString = "DEFAULT SEARCHING: GO LEFT" elif len(cnts) > 0: # Else if we are seeing some object... # Find the largest contour in the mask and use it to compute the minimum enclosing circle and centroid c = max(cnts, key=cv2.contourArea) ((x, y), radius) = cv2.minEnclosingCircle(c) M = cv2.moments(c) center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"])) filteredPtsRadius = [radius] # Only consider it to a valid object if it's big enough - else it could be some other random thing if filteredPtsRadius[0] <= 25: # TODO this is the same code as the block above - I should extract these out to a function # If we haven't seen the object before, then we'll stay halted until we see one # If we HAVE seen the object before, then we'll move in the direction (left or right) that we did # most recently if not objectSeenOnce: self.send_serial_command(Direction.STOP, b'h'); commandString = "STOP"; else: # Object has been seen before if leftOrRightLastSent is not None: if leftOrRightLastSent == Direction.RIGHT: self.send_serial_command(Direction.RIGHT, b'r'); commandString = "SEARCHING: GO RIGHT" elif leftOrRightLastSent == Direction.LEFT: self.send_serial_command(Direction.LEFT, b'l'); commandString = "SEARCHING: GO LEFT" else: # variable hasn't been set yet (seems unlikely), but default to left self.send_serial_command(Direction.LEFT, b'l'); commandString = "DEFAULT SEARCHING: GO LEFT" else: # This object isn't super small ... we should proceed with the tracking # Set objectSeenOnce to True if isn't already if not objectSeenOnce: objectSeenOnce = True # draw the circle on the frame TODO consider removing this eventually - could speed things up (barely) cv2.circle(frame, (int(x), int(y)), int(filteredPtsRadius[0]), (0, 255, 255), 2) cv2.circle(frame, center, 5, (0, 0, 255), -1) filteredPtsX = [center[0]] filteredPtsY = [center[1]] # Update PyGame Values if run_py_eyes: lefty = int(filteredPtsY[0] + 100) leftx = int(abs(filteredPtsX[0] - 600) / 2 + 106) width = int(filteredPtsRadius[0]) # Check radius and center of the blob to determine robot action # What actions should take priority? # 1. Moving Backward (only if it's super close) # 2. Moving Left/Right # 3. Moving Forward/Backward # Why? Because if we're too close any turn would be too extreme. We need to take care of that first if filteredPtsRadius[0] > radiusTooCloseLowerLimit: commandString = "MOVE BACKWARD - TOO CLOSE TO TURN" self.send_serial_command(Direction.BACKWARD, b'b') elif filteredPtsX[0] > centerRightBound: commandString = "GO RIGHT" self.send_serial_command(Direction.RIGHT, b'r') if leftOrRightLastSent != Direction.RIGHT: leftOrRightLastSent = Direction.RIGHT elif filteredPtsX[0] < centerLeftBound: commandString = "GO LEFT" self.send_serial_command(Direction.LEFT, b'l') if leftOrRightLastSent != Direction.LEFT: leftOrRightLastSent = Direction.LEFT elif filteredPtsRadius[0] < radiusInRangeLowerBound: commandString = "MOVE FORWARD" self.send_serial_command(Direction.FORWARD, b'f') elif filteredPtsRadius[0] > radiusInRangeUpperBound: commandString = "MOVE BACKWARD" self.send_serial_command(Direction.BACKWARD, b'b') elif radiusInRangeLowerBound < filteredPtsRadius[0] < radiusInRangeUpperBound: commandString = "STOP MOVING - IN RANGE" self.send_serial_command(Direction.STOP, b'h') # Put text on the camera image to display on the screen cv2.putText(frame, 'center coordinate: (' + str(filteredPtsX[0]) + ',' + str(filteredPtsY[0]) + ')', (10, 60), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) cv2.putText(frame, 'filtered radius: (' + str(filteredPtsRadius[0]) + ')', (10, 90), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) # The below steps are run regardless of whether we see a valid object or not ... # Show FPS (TODO delete this later) cv2.putText(frame, commandString, (10, 30), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) cv2.putText(frame, 'FPS: (' + str(fps) + ')', (10, 120), self.font, 0.5, (200, 255, 155), 1, cv2.LINE_AA) # show webcam video with object drawings on the screen cv2.imshow("result", frame) if run_py_eyes: # If our coordinates are out of bounds for the eyes, then cap them at their correct bounds if leftx < 106: leftx = 106 if leftx > 406: leftx = 406 if lefty < 150: lefty = 150 if lefty > 450: lefty = 450 # rightx = leftx + 400 + 112 - width # if rightx < 568: # rightx = 568 # if rightx > 968: # rightx = 968 # Note that the eyes could be made to get a little crossed eyed (close together) when you get very # close to the robot. It's not hard to do, but I didn't include it here (that's why the above lines # are commented out). # Draw left pupil rects.append(pygame.draw.circle(surface, self.RGB_BLACK, (leftx, lefty), self.EYEBALL_RADIUS, 0)) # Draw right pupil rects.append( pygame.draw.circle(surface, self.RGB_BLACK, (leftx + 500 + 12, lefty), self.EYEBALL_RADIUS, 0)) # Update the display so our changes show up pygame.display.update(rects) # Save the left and right pupil circles so that we can remove them on the next iteration (instead of # clearing the whole display and redrawing it all (which is expensive)) rects = [pygame.draw.circle(surface, self.RGB_WHITE, (leftx, lefty), self.EYEBALL_RADIUS, 0), pygame.draw.circle(surface, self.RGB_WHITE, (leftx + 500 + 12, lefty), self.EYEBALL_RADIUS, 0)] # Close application on 'q' key press, or if the queue is not empty (there's some command to respond to). key = cv2.waitKey(1) & 0xFF if (key == ord("q")) or (not cvQueue.empty()): self.send_serial_command(Direction.STOP, b'h') # We've been requested to leave ... # Don't destroy everything - just destroy cv2 windows ... webcam still runs cv2.destroyAllWindows() if run_py_eyes: pygame.display.quit() pygame.quit() break
# indices 3,4,5,6 store the bounding box coordinates in order [xmin, ymin, xmax, ymax] with values in the range 0-1 bbox = detections[0, 0, i, 3:7] * np.array( [orig_w, orig_h, orig_w, orig_h] ) # scaling bounding box coordinates back to original frame dimensions bbox = bbox.astype(np.int) # type casting and rounding to int type cv2.rectangle( frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 0, 255), 2) # drawing rectangular bounding boxes around detections return frame # setting up input video stream for reading from webcam webcam_stream = WebcamVideoStream( 0) # opening video stream from primary camera webcam_stream.start() fps = FPS() # for computing frames processed per second # processing video frames fps.start() while True: # reading next frame from input stream frame = webcam_stream.read() fps.update() # detecting faces in the read frame frame_with_detections = detect_faces(frame) # displaying the frame cv2.imshow('Detected Faces', frame_with_detections) key_pressed = cv2.waitKey(1) # a 1 millisecond delay
with model.as_default(): with tf.Session(graph=model) as sess: imageTensor = model.get_tensor_by_name("image_tensor:0") boxesTensor = model.get_tensor_by_name("detection_boxes:0") # for each bounding box we would like to know the score # (i.e., probability) and class label scoresTensor = model.get_tensor_by_name("detection_scores:0") classesTensor = model.get_tensor_by_name("detection_classes:0") numDetections = model.get_tensor_by_name("num_detections:0") drawboxes = [] # cap = cv2.VideoCapture(url) vs = WebcamVideoStream(src=0) vs.start() while True: frame = vs.read() if frame is None: continue frame = cv2.flip(frame, 1) image = frame (H, W) = image.shape[:2] # print("H,W:", (H, W)) output = image.copy() img_ff, bin_mask, res = ff.find_hand_old(image.copy()) image = cv2.cvtColor(res, cv2.COLOR_BGR2RGB) image = np.expand_dims(image, axis=0) (boxes, scores, labels, N) = sess.run( [boxesTensor, scoresTensor, classesTensor, numDetections],
class EyeCanSee(object): def __init__(self, center=int(cvsettings.CAMERA_WIDTH / 2), debug=False, is_usb_webcam=True, period_s=0.025): # Our video stream # If its not a usb webcam then get pi camera if not is_usb_webcam: self.vs = PiVideoStream(resolution=(cvsettings.CAMERA_WIDTH, cvsettings.CAMERA_HEIGHT)) # Camera cvsettings self.vs.camera.shutter_speed = cvsettings.SHUTTER self.vs.camera.exposure_mode = cvsettings.EXPOSURE_MODE self.vs.camera.exposure_compensation = cvsettings.EXPOSURE_COMPENSATION self.vs.camera.awb_gains = cvsettings.AWB_GAINS self.vs.camera.awb_mode = cvsettings.AWB_MODE self.vs.camera.saturation = cvsettings.SATURATION self.vs.camera.rotation = cvsettings.ROTATION self.vs.camera.video_stabilization = cvsettings.VIDEO_STABALIZATION self.vs.camera.iso = cvsettings.ISO self.vs.camera.brightness = cvsettings.BRIGHTNESS self.vs.camera.contrast = cvsettings.CONTRAST # Else get the usb camera else: self.vs = WebcamVideoStream(src=0) self.vs.stream.set(cv2.CAP_PROP_FRAME_WIDTH, cvsettings.CAMERA_WIDTH) self.vs.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, cvsettings.CAMERA_HEIGHT) # Has camera started self.camera_started = False self.start_camera() # Starts our camera # To calculate our error in positioning self.center = center # To determine if we actually detected lane or not self.detected_lane = False # debug mode on? (to display processed images) self.debug = debug # Time interval between in update (in ms) # FPS = 1/period_s self.period_s = period_s # Starting time self.start_time = time.time() # Mouse event handler for get_hsv def on_mouse(self, event, x, y, flag, param): if event == cv2.EVENT_LBUTTONDBLCLK: # Circle to indicate hsv location, and update frame cv2.circle(self.img_debug, (x, y), 3, (0, 0, 255)) cv2.imshow('hsv_extractor', self.img_debug) # Print values values = self.hsv_frame[y, x] print('H:', values[0], '\tS:', values[1], '\tV:', values[2]) def get_hsv(self): cv2.namedWindow('hsv_extractor') while True: self.grab_frame() # Bottom ROI cv2.rectangle( self.img_debug, (0, cvsettings.HEIGHT_PADDING_BOTTOM - 2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2) # Top ROI cv2.rectangle( self.img_debug, (0, cvsettings.HEIGHT_PADDING_TOP - 2), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT + 2), (0, 250, 0), 2) # Object cv2.rectangle( self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2) self.hsv_frame = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) # Mouse handler cv2.setMouseCallback('hsv_extractor', self.on_mouse, 0) cv2.imshow('hsv_extractor', self.img_debug) key = cv2.waitKey(0) & 0xFF if key == ord('q'): break self.stop_camera() cv2.destroyAllWindows() # Starts camera (needs to be called before run) def start_camera(self): self.camera_started = True self.vs.start() time.sleep(2.0) # Wait for camera to cool def stop_camera(self): self.camera_started = False self.vs.stop() # Grabs frame from camera def grab_frame(self): # Starts camera if it hasn't been started if not self.camera_started: self.start_camera() self.img = self.vs.read() self.img_debug = self.img.copy() # Normalizes our image def normalize_img(self): # Crop img and convert to hsv self.img_roi_bottom = np.copy(self.img[ cvsettings. HEIGHT_PADDING_BOTTOM:int(cvsettings.HEIGHT_PADDING_BOTTOM + cvsettings.IMG_ROI_HEIGHT), :]) self.img_roi_top = np.copy(self.img[ cvsettings.HEIGHT_PADDING_TOP:int(cvsettings.HEIGHT_PADDING_TOP + cvsettings.IMG_ROI_HEIGHT), :]) self.img_roi_bottom_hsv = cv2.cvtColor(self.img_roi_bottom, cv2.COLOR_BGR2HSV).copy() self.img_roi_top_hsv = cv2.cvtColor(self.img_roi_top, cv2.COLOR_BGR2HSV).copy() # Get our ROI's shape # Doesn't matter because both of them are the same shape self.roi_height, self.roi_width, self.roi_channels = self.img_roi_bottom.shape # Smooth image and convert to bianry image (threshold) # Filter out colors that are not within the RANGE value def filter_smooth_thres(self, RANGE, color): for (lower, upper) in RANGE: lower = np.array(lower, dtype='uint8') upper = np.array(upper, dtype='uint8') mask_bottom = cv2.inRange(self.img_roi_bottom_hsv, lower, upper) mask_top = cv2.inRange(self.img_roi_top_hsv, lower, upper) blurred_bottom = cv2.medianBlur(mask_bottom, 5) blurred_top = cv2.medianBlur(mask_top, 5) # Morphological transformation kernel = np.ones((2, 2), np.uint8) smoothen_bottom = blurred_bottom #cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5) smoothen_top = blurred_top # cv2.morphologyEx(blurred, cv2.MORPH_OPEN, kernel, iterations=5) """ if self.debug: cv2.imshow('mask bottom ' + color, mask_bottom) cv2.imshow('blurred bottom' + color, blurred_bottom) cv2.imshow('mask top ' + color, mask_top) cv2.imshow('blurred top' + color, blurred_top) """ return smoothen_bottom, smoothen_top # Gets metadata from our contours def get_contour_metadata(self): # Metadata (x,y,w,h)for our ROI contour_metadata = {} for cur_img in [ self.thres_yellow_bottom, self.thres_yellow_top, self.thres_blue_bottom, self.thres_blue_top ]: key = '' # Blue is left lane, Yellow is right lane if cur_img is self.thres_yellow_bottom: key = 'right_bottom' elif cur_img is self.thres_yellow_top: key = 'right_top' elif cur_img is self.thres_blue_bottom: key = 'left_bottom' elif cur_img is self.thres_blue_top: key = 'left_top' _, contours, hierarchy = cv2.findContours(cur_img.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cur_height, cur_width = cur_img.shape # Get index of largest contour try: areas = [cv2.contourArea(c) for c in contours] max_index = np.argmax(areas) cnt = contours[max_index] # Metadata of contour x, y, w, h = cv2.boundingRect(cnt) # Normalize it to the original picture x += int(cvsettings.WIDTH_PADDING + w / 2) if 'top' in key: y += int(cvsettings.HEIGHT_PADDING_TOP + h / 2) else: y += int(cvsettings.HEIGHT_PADDING_BOTTOM + h / 2) contour_metadata[key] = (x, y) self.detected_lane = True # If it throws an error then it doesn't have a ROI # Means we're too far off to the left or right except: # Blue is left lane, Yellow is right lane x = int(cvsettings.WIDTH_PADDING) - cvsettings.CAMERA_WIDTH if 'bottom' in key: y = int(cur_height / 2) + int(cvsettings.HEIGHT_PADDING_BOTTOM + cur_height / 2) else: y = int( cur_height / 2) + int(cvsettings.HEIGHT_PADDING_TOP + cur_height / 2) if 'right' in key: x = int(cur_width) * 2 contour_metadata[key] = (x, y) self.detected_lane = False return contour_metadata # Gets the centered coord of the detected lines def get_centered_coord(self): bottom_centered_coord = None top_centered_coord = None left_xy_bottom = self.contour_metadata['left_bottom'] right_xy_bottom = self.contour_metadata['right_bottom'] left_xy_top = self.contour_metadata['left_top'] right_xy_top = self.contour_metadata['right_top'] bottom_xy = (left_xy_bottom[0] + right_xy_bottom[0], left_xy_bottom[1] + right_xy_bottom[1]) bottom_centered_coord = (int(bottom_xy[0] / 2), int(bottom_xy[1] / 2)) top_xy = (left_xy_top[0] + right_xy_top[0], left_xy_top[1] + right_xy_top[1]) top_centered_coord = (int(top_xy[0] / 2), int(top_xy[1] / 2)) # Left can't be greater than right and vice-versa if left_xy_top > right_xy_top: top_centered_coord = (0, top_centered_coord[1]) elif right_xy_top < left_xy_top: top_centered_corrd = (cvsettings.CAMERA_WIDTH, top_centered_coord[1]) if left_xy_bottom > right_xy_bottom: bottom_centered_coord = (0, bottom_centered_coord[1]) elif right_xy_bottom < left_xy_bottom: bottom_centered_coord = (cvsettings.CAMERA_WIDTH, top_centered_coord[1]) return bottom_centered_coord, top_centered_coord # Gets the error of the centered coordinates (x) # Also normlizes their values def get_errors(self): top_error = (float(self.center_coord_top[0]) - float(self.center)) / float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) bottom_error = (float(self.center_coord_bottom[0]) - float(self.center)) / float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) relative_error = (float(self.center_coord_top[0]) - float( self.center_coord_bottom[0])) / float(cvsettings.CAMERA_WIDTH + cvsettings.WIDTH_PADDING) return (top_error + relative_error + bottom_error) / 3.0 # Object avoidance def where_object_be(self): # We only want to detect objects in our path: center of top region and bottom region # So to save processing speed, we'll only threshold from center of top region to center of bottom region left_x = cvsettings.WIDTH_PADDING right_x = cvsettings.CAMERA_WIDTH # Image region with objects img_roi_object = self.img[cvsettings.OBJECT_HEIGHT_PADDING:int( cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), left_x:right_x] img_roi_object_hsv = cv2.cvtColor(img_roi_object, cv2.COLOR_BGR2HSV).copy() # Filtering color and blurring for (lower, upper) in cvsettings.OBJECT_HSV_RANGE: lower = np.array(lower, dtype='uint8') upper = np.array(upper, dtype='uint8') mask_object = cv2.inRange(img_roi_object_hsv, lower, upper) blurred_object = cv2.medianBlur(mask_object, 25) # Finding position of object (if its there) _, contours, hierarchy = cv2.findContours(blurred_object.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) left_x = self.contour_metadata['left_top'][0] right_x = self.contour_metadata['right_top'][0] for c in contours: areas = cv2.contourArea(c) # Object needs to be larger than a certain area if areas > cvsettings.OBJECT_AREA: x, y, w, h = cv2.boundingRect(c) y += int(cvsettings.OBJECT_HEIGHT_PADDING + h / 2) # Confusing part - this finds the object and makes it think that # it is also a line to avoid bumping into it # It +w and -w to find which line its closest to and then set # the opposite as to be the new left/right lane # e.g. line is closest to left lane (x-w), so left lane new x is # (x+w) distance_to_left = abs(x - left_x) distance_to_right = abs(x + w - right_x) # Make object's left most area the middle of right lane if distance_to_left > distance_to_right: self.contour_metadata['right_top'] = ( x, self.contour_metadata['right_top'][1]) # Make object's right most area the middle of left lane elif distance_to_right > distance_to_right: self.contour_metadata['left_top'] = ( x + w, self.contour_metadata['left_top'][1]) if self.debug: cv2.circle(self.img_debug, (x + (w / 2), y + (h / 2)), 5, (240, 32, 160), 2) if self.debug: cv2.imshow('Blurred object', blurred_object) # Where are we relative to our lane def where_lane_be(self): # Running once every period_ms while float(time.time() - self.start_time) < self.period_s: pass # Update time instance self.start_time = time.time() # Camera grab frame and normalize it self.grab_frame() self.normalize_img() # Filter out them colors self.thres_blue_bottom, self.thres_blue_top = self.filter_smooth_thres( cvsettings.BLUE_HSV_RANGE, 'blue') self.thres_yellow_bottom, self.thres_yellow_top = self.filter_smooth_thres( cvsettings.YELLOW_HSV_RANGE, 'yellow') # Get contour meta data self.contour_metadata = self.get_contour_metadata() # Finds objects and (and corrects lane position) # this overwrite contour_metadata #self.where_object_be() # Find the center of the lanes (bottom and top) [we wanna be in between] self.center_coord_bottom, self.center_coord_top = self.get_centered_coord( ) # Gets relative error between top center and bottom center self.relative_error = self.get_errors() if self.debug: # Drawing locations blue_top_xy = self.contour_metadata['left_top'] blue_bottom_xy = self.contour_metadata['left_bottom'] yellow_top_xy = self.contour_metadata['right_top'] yellow_bottom_xy = self.contour_metadata['right_bottom'] # Circles to indicate lanes cv2.circle(self.img_debug, blue_top_xy, 5, (255, 0, 0), 2) cv2.circle(self.img_debug, blue_bottom_xy, 5, (255, 0, 0), 2) cv2.circle(self.img_debug, yellow_top_xy, 5, (0, 255, 255), 2) cv2.circle(self.img_debug, yellow_bottom_xy, 5, (0, 255, 255), 2) cv2.circle(self.img_debug, self.center_coord_bottom, 5, (0, 255, 0), 3) cv2.circle(self.img_debug, self.center_coord_top, 5, (0, 255, 0), 3) # ROI for object detection cv2.rectangle( self.img_debug, (0, cvsettings.OBJECT_HEIGHT_PADDING), (cvsettings.CAMERA_WIDTH, cvsettings.HEIGHT_PADDING_TOP - cvsettings.OBJECT_HEIGHT_PADDING), (238, 130, 238), 2) # Displaying image cv2.imshow('img', self.img_debug) #cv2.imshow('img_roi top', self.img_roi_top) #cv2.imshow('img_roi bottom', self.img_roi_bottom) #cv2.imshow('img_hsv', self.img_roi_hsv) cv2.imshow('thres_blue_bottom', self.thres_blue_bottom) cv2.imshow('thres_blue_top', self.thres_blue_top) cv2.imshow('thres_yellow_bottom', self.thres_yellow_bottom) cv2.imshow('thres_yellow_top', self.thres_yellow_top) key = cv2.waitKey( 0) & 0xFF # Change 1 to 0 to pause between frames # Use this to calculate fps def calculate_fps(self, frames_no=100): fps = FPS().start() # Don't wanna display window if self.debug: self.debug = not self.debug for i in range(0, frames_no): self.where_lane_be() fps.update() fps.stop() # Don't wanna display window if not self.debug: self.debug = not self.debug print('Time taken: {:.2f}'.format(fps.elapsed())) print('~ FPS : {:.2f}'.format(fps.fps())) # Use this to save images to a location def save_images(self, dirname='dump'): import os img_no = 1 # Makes the directory if not os.path.exists('./' + dirname): os.mkdir(dirname) while True: self.grab_frame() if self.debug: cv2.imshow('frame', self.img) k = cv2.waitKey(1) & 0xFF if k == ord('s'): cv2.imwrite( os.path.join(dirname, 'dump_' + str(img_no) + '.jpg'), self.img) img_no += 1 elif k == ord('q'): break cv2.destroyAllWindows() # Destructor def __del__(self): self.vs.stop() cv2.destroyAllWindows()
class VideoStreamer: outputFrame = None lock = None app = None vs = None PORT = '8000' app = Flask('video_stream') def __init__(self): self.stream = False self.app.add_url_rule('/video_on', 'video_on', self.start_new_stream) self.app.add_url_rule('/video_off', 'video_off', self.stop_stream) self.app.add_url_rule('/', 'index', index) self.app.add_url_rule('/video', 'video', self.video_feed) self.app.run(host='0.0.0.0', port=self.PORT, threaded=True, use_reloader=False) def start_new_stream(self): # initialize the output frame and a lock used to ensure thread-safe # exchanges of the output frames (useful when multiple browsers/tabs # are viewing the stream) print('starting new stream') self.outputFrame = None self.lock = threading.Lock() # initialize a flask object # initialize the video stream and allow the camera sensor to warmup print('pre initialize') try: self.vs = WebcamVideoStream(src=0) except: self.vs = WebcamVideoStream(src=-1) print(self.vs) self.vs.start() time.sleep(2.0) self.t = threading.Thread(target=self.get_frames, args=(32,)) self.t.daemon = True self.t.start() # start the flask app self.stream = True print('new_stream started') return 'video streaming' def stop_stream(self): # release the video stream pointer self.stream = False self.vs.stream.release() self.vs.stop() print('stream stopped') return 'video stopped' def get_frames(self,frameCount): # grab global references to the video stream, output frame, and # lock variables total = 0 print('entered thread') # loop over frames from the video stream while self.stream: print(total) # read the next frame from the video stream, resize it, # convert the frame to grayscale, and blur it frame = self.vs.read() frame = imutils.resize(frame, width=400) frame = cv2.flip(frame, 0) # frame = imutils.resize(frame, width=400) # grab the current timestamp and draw it on the frame timestamp = datetime.datetime.now() if frame is not None: cv2.putText(frame, timestamp.strftime( "%A %d %B %Y %I:%M:%S%p"), (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1) total += 1 with self.lock: self.outputFrame = frame.copy() def generate(self): # grab global references to the output frame and lock variables # loop over frames from the output stream while True: # wait until the lock is acquired with self.lock: # check if the output frame is available, otherwise skip # the iteration of the loop if self.outputFrame is None: continue # encode the frame in JPEG format (flag, encodedImage) = cv2.imencode(".jpg", self.outputFrame) # ensure the frame was successfully encoded if not flag: continue # yield the output frame in the byte format yield (b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(encodedImage) + b'\r\n') def video_feed(self): # return the response generated along with the specific media # type (mime type) if self.stream: return Response(self.generate(), mimetype="multipart/x-mixed-replace; boundary=frame") else: return 'no video available'
# The following was used for troubleshooting # cv2.imshow("Calibration Image",image) # Marker = the number of pixels that equate to the width marker = find_marker(edged) focalLength = (marker[1][0] * KNOWN_DISTANCE) / KNOWN_WIDTH print("Pixcel Width", marker[1][0]) print("Focal Length", focalLength) inches = distance_to_camera(KNOWN_WIDTH, focalLength, marker[1][0]) print("Distance", inches) print("\nInitalizing camera in multi-thread mode") print("Initalizing FPS stats collector in multi-thread mode\n") camera = WebcamVideoStream(src=0).start() camera.start() sleep(2) print("Camera Active: ", camera.grabbed) while not camera.grabbed: print("Restarting Camera...") camera = WebcamVideoStream(src=0).start() sleep(5) fps = FPS().start() camera.start() getFrame = camera.read() print(" Cam Width:", getFrame.shape[1]) print("cam Height:", getFrame.shape[0])
green_value_max = 174 green_hsv_max = np.array( [green_hue_max, green_saturation_max, green_value_max]) mm.append( MM.M_and_M(color_id, "Green", max_age, greenCircleColor, green_hsv_min, green_hsv_max, greenTextLocation)) # Create Green Tracking Object ########## # Create video camera object and start streaming to it. ########## #cap = cv2.VideoCapture(0) # NONTHREAD 0 == Continuous stream capvs = WebcamVideoStream( src=0) # THREAD Create video capture in separte thread capvs.start() # set up visual imaging for the trigger line that is the count line when crossed cnt_down = 0 y_trigger = 160 trigger_line_color = (255, 0, 0) trigger_line = np.array([[0, y_trigger], [720, y_trigger]]) counter = 0 # Debug variables used to see how long it takes to process a frame. millis1 = 0 millis2 = 0 milli_total = 0 fps_count = 1
video = int(video) if str.isdigit(video) else video # initialize the video stream video_stream = WebcamVideoStream(src=video) # 4:3=1280x960, 640x480 / 16:9=1280x720, 640x360 video_stream.stream.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) video_stream.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) video_stream.stream.set(cv2.CAP_PROP_FPS, 24) print('\nVideo_stream.stream.get') print(video_stream.stream.get(cv2.CAP_PROP_FRAME_WIDTH)) print(video_stream.stream.get(cv2.CAP_PROP_FRAME_HEIGHT)) print(video_stream.stream.get(cv2.CAP_PROP_FPS), '\n') video_stream.start() # a camera warmup time of 2.0 seconds frame = video_stream.read() # crop_y to 360 if frame.shape == (480, 640, 3): frame = frame[60:-60, :] print('Video Size is', frame.shape) print('Streaming frames to the server...') while True: # read the frame from the camera and send it to the server frame = video_stream.read() # crop_y to 360 if frame.shape == (480, 640, 3): frame = frame[60:-60, :]