def __init__(self, module_name, settings, pipe): """Initialize video capture, logger, and processing plugins. Args: module_name -- TODO settings -- dict of settings for this object. pipe -- TODO """ self.module_name = module_name self.master_settings = settings['sensor/vision/control'] self.settings = settings[module_name] self.processor = FrameProcessor() self._com = Communicator(module_name) # Nagging messages from the parent come from here. The process must # respond quickly enough or be killed by the parent. self._pipe = pipe self._fresh_frame_available = Event() self._got_im, self._im = None, None self._init_stream() # Individual stream images will be processed by modules stored here. self._plugins = [] # Load, instantiate, and store the modules defined in the settings file. for plugin_name in self.settings['plugins']: # Name of module and the class should be the same. module_obj = getattr(import_module( '..'+plugin_name, package='plugins.subpkg'), plugin_name)(self.processor, self.settings) self._plugins += [module_obj] # Configure the VideoCapture object. self._vision_process()
class StreamProcessor(object): """Open a video stream and process frames. Report finds to the grapevine.""" def __init__(self, module_name, settings, pipe): """Initialize video capture, logger, and processing plugins. Args: module_name -- TODO settings -- dict of settings for this object. pipe -- TODO """ self.module_name = module_name self.master_settings = settings['sensor/vision/control'] self.settings = settings[module_name] self.processor = FrameProcessor() self._com = Communicator(module_name) # Nagging messages from the parent come from here. The process must # respond quickly enough or be killed by the parent. self._pipe = pipe self._fresh_frame_available = Event() self._got_im, self._im = None, None self._init_stream() # Individual stream images will be processed by modules stored here. self._plugins = [] # Load, instantiate, and store the modules defined in the settings file. for plugin_name in self.settings['plugins']: # Name of module and the class should be the same. module_obj = getattr(import_module( '..'+plugin_name, package='plugins.subpkg'), plugin_name)(self.processor, self.settings) self._plugins += [module_obj] # Configure the VideoCapture object. self._vision_process() def _init_stream(self): """Initialize the video stream.""" if self.settings['stream_type'] == 'device': cam_path = os.path.realpath(self.settings['symlink']) # Check that the symlink points to something that really exists. if '/dev/video' not in cam_path: raise Exception('Broken symlink {sym}'.format( sym=self.settings['symlink'])) self._cap = cv2.VideoCapture(int(cam_path[-1])) self._cap.set(cv.CV_CAP_PROP_FPS, self.settings['fps']) self._frame_grabber_thread = Thread(target=self.refresh_frame) self._frame_grabber_thread.daemon = True self._frame_grabber_thread.start() #self._frame_grabber_thread = None elif self.settings['stream_type'] == 'recorded_video': # The 'symlink' key didn't exist. This must be a test with recorded # video. self._cap = cv2.VideoCapture(self.settings['recorded_video']) self._frame_grabber_thread = None else: raise Exception( 'No stream type specified for {module_name}'.format( module_name=module_name)) def _get_frame(self): """Get the most recent frame from the video stream.""" if self._frame_grabber_thread is not None: self._fresh_frame_available.wait() return (self._got_im, self._im) else: return self._cap.read() def refresh_frame(self): """Get and store latest frame from video stream as fast as possible.""" failed_count = 0 # Assuming 30fps, 150 fails in a row means 5s of no video. while failed_count < self.master_settings['max_failed_frames']: self._got_im, self._im = self._cap.read() self._fresh_frame_available.set() self._fresh_frame_available.clear() if not self._got_im: failed_count += 1 else: failed_count = 0 raise Exception('Camera sucks') def _vision_process(self): """Detect obstacle details from video stream. Report findings.""" while True: start_time = time() got_im, im = self._get_frame() #print 'new frame' if self._pipe.poll() is True: # Send the message back to parent to prove this process # isn't frozen. self._pipe.send(self._pipe.recv()) if got_im: # Check if parent sent a message over the pipe. self.processor.load_im(im) # self.processor.hacky_display() # FIXME this needs to go away. packet = {} for plugin in self._plugins: plugin.process_image(packet) self._com.publish_message(packet)
def main(): args = parse_args() FOLDER = args.output_folder FRAME_LIMIT = args.frame_limit VIDEO_FILE = args.video_file DRAW_BOXES = args.draw_boxes SCORES_FILE = args.scores_file DETECTION = args.detection GAZE_ESTIMATION = not args.no_gaze_estimation OUTPUT_TYPE = args.output_type NO_SCORES = args.no_scores if GAZE_ESTIMATION: processor = FrameProcessor() else: processor = None if DETECTION == "cv2": detector = CVFaceDetector() elif DETECTION == "cnn": detector = MTCNNFaceDetector() else: raise Exception(f"detection method {DETECTION} is invalid, expected one of [cv2, cnn]") df_faces = [] if NO_SCORES: print(f">> Reading frames from {VIDEO_FILE}") frames = [ cv2.imread(os.path.join(VIDEO_FILE, frame_file)) for frame_file in os.listdir(VIDEO_FILE) ] h, w, _ = frames[0].shape df = pd.DataFrame([{ 'track_id': i // 128, 'frame_id': i, 'abs_frame_id': i, 'box_id': 0, 'x': 0, 'y': 0, 'x2': w, 'y2': h, } for i in range(len(frames))]) else: print(f">> Reading video from {VIDEO_FILE}") df = pd.read_csv(SCORES_FILE, header=None, index_col=None, names=[ "track_id", "frame_id", "box_id", "x", "y", "x2", "y2"] + list(range(80))) df["abs_frame_id"] = df.frame_id + df.track_id - 128 frames = collect_frames(VIDEO_FILE, FRAME_LIMIT) if os.path.isdir(FOLDER): shutil.rmtree(FOLDER) print(">> Matching frames") for i, frame in tqdm(enumerate(frames), total=len(frames)): new_frame = match_faces_bodies( frame, df[df.abs_frame_id == i], processor=processor, detector=detector, draw_boxes=DRAW_BOXES ) frames[i], new_df_faces = new_frame df_faces += new_df_faces if not os.path.isdir(FOLDER): print(f">> Creating folder {FOLDER}") os.makedirs(FOLDER) if OUTPUT_TYPE == "frames": os.makedirs(os.path.join(FOLDER, "frames")) save_video_to_files(frames, os.path.join(FOLDER, "frames")) df_faces = pd.DataFrame(df_faces) print(">> Saving box files") box_ids = df_faces.box_id.unique() for box_id in tqdm(box_ids, total=len(box_ids)): with open(os.path.join(FOLDER, f"person{box_id}.txt"), "w") as f: for _, frame in df_faces[df_faces.box_id == box_id].iterrows(): bbox = str([frame.x, frame.y, frame.x2, frame.y2]).replace(" ", "").strip("[]") f.write(f"frame_{frame.frame_id}.png,{bbox}\n") else: output_video_file(os.path.join(FOLDER, "video.mp4"), frames)
agent = PG(run_name=arg.run_name, input_shape=[160,160], n_action=3, learning_rate=arg.learning_rate, save_path=arg.save_path, record_io=False, record=True, gpu_fraction=0.9) for i in range(arg.games_num): replay_buffer = ReplayBuffer(input_shape=[160, 160], start_size=32, max_size=10000000) memory = [] total_reward = 0 obs = env.reset() frame_processor = FrameProcessor(obs) obs, reward, done, _ = env.step(agent.random_action() + 1) # do random action at the first frame total_reward += reward # play one game while not done: input_frame = frame_processor.process(obs) prob = agent.get_action_prob(input_frame) action = np.random.choice(3, p=prob) obs, reward, done, _ = env.step(action + 1) if reward == 0: replay_buffer.store_transition(input_frame, action) else: total_reward += reward if reward == 1:
class StreamProcessor(object): """Open a video stream and process frames. Report finds to the grapevine.""" def __init__(self, module_name, settings, pipe): """Initialize video capture, logger, and processing plugins. Args: module_name -- TODO settings -- dict of settings for this object. pipe -- TODO """ self.module_name = module_name self.master_settings = settings['sensor/vision/control'] self.settings = settings[module_name] self.processor = FrameProcessor() self._com = Communicator(module_name) # Nagging messages from the parent come from here. The process must # respond quickly enough or be killed by the parent. self._pipe = pipe self._fresh_frame_available = Event() self._got_im, self._im = None, None self._init_stream() # Individual stream images will be processed by modules stored here. self._plugins = [] # Load, instantiate, and store the modules defined in the settings file. for plugin_name in self.settings['plugins']: # Name of module and the class should be the same. module_obj = getattr( import_module('..' + plugin_name, package='plugins.subpkg'), plugin_name)(self.processor, self.settings) self._plugins += [module_obj] # Configure the VideoCapture object. self._vision_process() def _init_stream(self): """Initialize the video stream.""" if self.settings['stream_type'] == 'device': cam_path = os.path.realpath(self.settings['symlink']) # Check that the symlink points to something that really exists. if '/dev/video' not in cam_path: raise Exception('Broken symlink {sym}'.format( sym=self.settings['symlink'])) self._cap = cv2.VideoCapture(int(cam_path[-1])) self._cap.set(cv.CV_CAP_PROP_FPS, self.settings['fps']) self._frame_grabber_thread = Thread(target=self.refresh_frame) self._frame_grabber_thread.daemon = True self._frame_grabber_thread.start() #self._frame_grabber_thread = None elif self.settings['stream_type'] == 'recorded_video': # The 'symlink' key didn't exist. This must be a test with recorded # video. self._cap = cv2.VideoCapture(self.settings['recorded_video']) self._frame_grabber_thread = None else: raise Exception( 'No stream type specified for {module_name}'.format( module_name=module_name)) def _get_frame(self): """Get the most recent frame from the video stream.""" if self._frame_grabber_thread is not None: self._fresh_frame_available.wait() return (self._got_im, self._im) else: return self._cap.read() def refresh_frame(self): """Get and store latest frame from video stream as fast as possible.""" failed_count = 0 # Assuming 30fps, 150 fails in a row means 5s of no video. while failed_count < self.master_settings['max_failed_frames']: self._got_im, self._im = self._cap.read() self._fresh_frame_available.set() self._fresh_frame_available.clear() if not self._got_im: failed_count += 1 else: failed_count = 0 raise Exception('Camera sucks') def _vision_process(self): """Detect obstacle details from video stream. Report findings.""" while True: start_time = time() got_im, im = self._get_frame() #print 'new frame' if self._pipe.poll() is True: # Send the message back to parent to prove this process # isn't frozen. self._pipe.send(self._pipe.recv()) if got_im: # Check if parent sent a message over the pipe. self.processor.load_im(im) # self.processor.hacky_display() # FIXME this needs to go away. packet = {} for plugin in self._plugins: plugin.process_image(packet) self._com.publish_message(packet)