def test_ffmpeg(self): """Runs two FFmpeg subprocesses: the first encodes raw 24-bit frames into raw MPEG-4 video stream, the second decodes that stream and feeds the reader again with raw 24-bit frames. The latter triggers detection of simple shapes on an image. The number of shapes detected is counted and signals to end the test. """ width = 480 height = 360 encoder_frame_buffer = FrameBuffer(10, width, height) decoder_frame_buffer = FrameBuffer(10, width, height) encoder_frame_queue = Queue(1) decoder_frame_queue = Queue(1) artist_subscribe_queue = Queue(1) decoder_subscribe_queue = Queue(1) log_queue = Queue() getLogger().addHandler(QueueHandler(log_queue)) stop_process_event = Event() latch = CountDownLatch(100) encoder = FFmpegEncoder("encoder", stop_process_event, log_queue, encoder_frame_queue, encoder_frame_buffer, ['ffmpeg', '-hide_banner', '-loglevel', 'panic', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', '{}x{}'.format(width, height), '-i', '-', '-an', '-f', 'm4v', '-'], None, PIPE) decoder = FFmpegDecoder("decoder", stop_process_event, log_queue, decoder_frame_queue, decoder_frame_buffer, ['ffmpeg', '-hide_banner', '-loglevel', 'panic', '-f', 'm4v', '-i', '-', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-'], None, PIPE) artist = Artist("artist", stop_process_event, log_queue, encoder_frame_queue, encoder_frame_buffer) conductor = WorkPublish(Thread, "conductor", stop_process_event, log_queue, artist_subscribe_queue, encoder_frame_buffer) processes = [LogHandler(Thread, "logger", stop_process_event, log_queue, filename=None), artist, conductor, encoder, decoder, Copy(Thread, "copier", stop_process_event, log_queue, encoder.stdout, decoder.stdin), ShapeDetector(Process, "detector", stop_process_event, log_queue, decoder_frame_queue, decoder_frame_buffer), ShapeCounter(Thread, "counter", stop_process_event, log_queue, decoder_subscribe_queue, decoder_frame_buffer, latch)] artist.subscribe(artist_subscribe_queue) decoder.subscribe(decoder_subscribe_queue) for process in processes: process.start() try: self.assertTrue(latch.wait(15)) finally: stop_process_event.set() for process in processes: process.join(30)
def test_frame_buffer(self): frame_buffer = FrameBuffer(10, 1, 1, 1) length = len(frame_buffer.frames) frame_cycle = cycle(range(length)) for i in range(-1, length * 3): _, frame_index = frame_buffer.select_next_ready(i) self.assertEqual(next(frame_cycle), frame_index)
def test_frame_buffer_expire(self): getLogger(FrameBuffer.__name__).setLevel(ERROR) frame_buffer = FrameBuffer(1, 1, 1, 1) for frame in frame_buffer.frames: frame.header.epoch = time() frame.latch.next() frame, frame_index = frame_buffer.select_next_ready() self.assertIsNone(frame) for frame in frame_buffer.frames: frame.header.epoch -= 60 frame, frame_index = frame_buffer.select_next_ready() self.assertIsNotNone(frame)
def _new_frame(self, frame_in, payload: Payload, stop_event, frame_buffer_in: FrameBuffer, frame_buffer_out: FrameBuffer, *args, **kwargs): self._subscribers_lock.acquire() try: max_subscribers = len(self._subscribers) if max_subscribers == 0: # Release the frame as no one has subscribed return frame_out, frame_index = frame_buffer_out.select_next_ready( self.__last_frame_index) self.__last_frame_index = frame_index if frame_out is None: raise BufferError self._incoming_frame(frame_in, frame_out, stop_event, *args, **kwargs) # Enforcing confirmation from all subscribers before returning frame into buffer frame_out.latch.next() frame_out.latch.next(max_subscribers) payload = Payload(self.name, frame_index) count = self._publish(payload) # Helps to return frame into buffer if not all subscribers received it while count < max_subscribers: frame_out.latch.next() count += 1 finally: self._subscribers_lock.release()
def _next_frame(self, frame_queue, frame_buffer: FrameBuffer, *args, **kwargs): frame, frame_index = frame_buffer.select_next_ready(self.__last_frame_index) self.__last_frame_index = frame_index if frame is None: raise BufferError return frame_index if self._new_frame(frame, frame_queue, frame_buffer, *args, **kwargs) else None
def _create_encoder(self, camera_config, camera_name, frame_buffer_out, buffer_size, detection_sieve, visual_effects, visual_effects_queue): if 'encoder' not in camera_config['ffmpeg']: return None, None, None encoder_queue = Queue(1) encoder = FFmpegEncoder(camera_name, self._stop_events[0], self._log_queue, encoder_queue, frame_buffer_out, camera_config['ffmpeg']['encoder'], self._config_path, DEVNULL if 'output' in camera_config else PIPE, kwargs={'log_level': self._args.log_level}) self._processes.append(encoder) detection_sieve.subscribe(visual_effects_queue) visual_effects.subscribe(encoder_queue) if 'output' in camera_config: return encoder, None, None mpegts_buffer = FrameBuffer(buffer_size, int(camera_config['width'] / 4), 188, 1) mpegts_reader = MpegTSReader(camera_name, self._stop_events[0], self._log_queue, encoder.stdout, mpegts_buffer, kwargs={'log_level': self._args.log_level}) self._processes.append(mpegts_reader) return encoder, mpegts_reader, mpegts_buffer
def __init_frame_keeper(self, camera_config): self.__dict = {} frame_index = 0 for entry in camera_config['detect']: coco_class = next(iter(entry)) idx = COCO_CLASSES.index(coco_class) self.__dict[idx] = KeepData(frame_index, 0, 0) frame_index += 1 self.__frame_keeper = FrameBuffer(frame_index, camera_config['width'], camera_config['height'])
def test_shape_detection(self): """Tests TensorFlow object detection using the trained model of simple geometric shapes. The detector recognises shapes drawn on a frame image, the sieve filters those having confidence above 50%, lastly shapes are counted signalling to end the test. """ with EnvironmentVarGuard() as env: env.set("TF_CPP_MIN_LOG_LEVEL", "3") env.set("CORAL_VISIBLE_DEVICES", "") env.set("CUDA_VISIBLE_DEVICES", "") frame_buffer = FrameBuffer(10, 100, 100) frame_queue = Queue(1) subscriber_queue = Queue(1) detection_sieve_queue = Queue(1) log_queue = Queue() getLogger().addHandler(QueueHandler(log_queue)) stop_process_event = Event() latch = CountDownLatch(100) artist = Artist("artist", stop_process_event, log_queue, frame_queue, frame_buffer) detection_sieve = DetectionSieve("sieve", stop_process_event, log_queue, detection_sieve_queue, frame_buffer, self._create_filters(), RateLimiter()) processes = [artist, detection_sieve, LogHandler(Thread, "logger", stop_process_event, log_queue, filename=None), ShapeCounter(Thread, "counter", stop_process_event, log_queue, subscriber_queue, frame_buffer, latch)] processes += create_object_detectors(Process, stop_process_event, log_queue, frame_queue, {artist.name: frame_buffer}, self._get_model_path()) artist.subscribe(detection_sieve_queue) detection_sieve.subscribe(subscriber_queue) for process in processes: process.start() try: self.assertTrue(latch.wait(15)) finally: stop_process_event.set() for process in processes: process.join(30)
def test_numpy_stream(self): """Tests shared memory usage across processes. One process fills a frame buffer with random data, while another performs simple math operations on the given random image, comparing with predicted result. """ frame_buffer = FrameBuffer(5, 10, 10, 1, 'd') frame_queue = Queue() log_queue = Queue() getLogger().addHandler(QueueHandler(log_queue)) stop_process_event = Event() log_handler = LogHandler(Process, "logger", stop_process_event, log_queue, filename=None) reader = NumpyRead(Process, "reader", stop_process_event, log_queue, frame_queue, frame_buffer) worker = NumpyWork(Process, "worker", stop_process_event, log_queue, frame_queue, frame_buffer) log_handler.start() reader.start() worker.start() try: # Wait till the last frame in the buffer is used. self.assertTrue(frame_buffer.frames[-1].latch.wait( State.PUBLISH, 5)) finally: stop_process_event.set() reader.join(30) worker.join(30) log_handler.join(30) self.assertEqual(len(frame_buffer.frames), frame_buffer.status[State.PUBLISH]) self.assertEqual(first=len(frame_buffer.frames), second=worker.matches, msg="Not all frames were processed")
def _next_frame(self, frame_queue, stop_event, frame_buffer: FrameBuffer): frame, frame_index = frame_buffer.select_next_ready( self._last_frame_index) self._last_frame_index = frame_index if frame is None: return None image_shape, image_np = frame.get_numpy_image() # Fill image buffer with random data data = np.random.randn(*image_shape) np.copyto(image_np, data) # Calculate sum of all pixels and do math against the sum x = np.sum(image_np[:]).item() y = random.random() operation = OPERATIONS[random.randint(0, len(OPERATIONS[0]))] expected_result = simple_math(operation, x, y) frame.header.epoch = time() frame.latch.next() return Action(frame_index, operation, y, expected_result)
def test_snapshot(self): width = 100 height = 100 frame_buffer = FrameBuffer(10, width, height) frame_queue = Queue(1) log_queue = Queue() getLogger().addHandler(QueueHandler(log_queue)) stop_process_event = Event() effect = MockEffect() effect.draw_rect = MagicMock() snapshot = Snapshot("snapshot", stop_process_event, log_queue, frame_queue, frame_buffer, self._create_detect_config(width, height), [effect]) processes = [ snapshot, LogHandler(Thread, "logger", stop_process_event, log_queue, filename=None) ] for process in processes: process.start() try: frame_index = 0 frame = frame_buffer.frames[frame_index] frame.header.detections[0].label = COCO_CLASSES.index('book') frame.header.detections[0].bounding_box.x_min = 1 frame.header.detections[0].bounding_box.y_min = 2 frame.header.detections[0].bounding_box.x_max = 3 frame.header.detections[0].bounding_box.y_max = 4 frame.header.epoch = time() frame.latch.next() frame.latch.next() payload = Payload(None, frame_index) frame_queue.put(payload) self.assertTrue( frame.latch.wait_for(State.READY, stop_process_event.is_set, 10)) with self.assertRaises(AssertionError): snapshot.get('person') self.assertIsNotNone(snapshot.get('book')) effect.draw_rect.assert_called_with(1, 2, 3, 4) finally: stop_process_event.set() for process in processes: process.join(30)
def prepare_shape_model(groups): frame_buffer = FrameBuffer(10, 300, 300) frame_queue = Queue(1) subscriber_queue = Queue(1) log_queue = CountableQueue() getLogger().addHandler(QueueHandler(log_queue)) stop_logging_event = Event() log_handler = LogHandler(Thread, "logger", stop_logging_event, log_queue, filename=None) log_handler.start() for group, count in groups.items(): path = os.path.abspath( os.path.join( Path(__file__).parent.parent.parent.parent, 'build/test/model')) os.makedirs(os.path.join(path, "images", group), exist_ok=True) os.makedirs(os.path.join(path, "annotations"), exist_ok=True) stop_process_event = Event() latch = CountDownLatch(count) artist = Artist("artist", stop_process_event, log_queue, frame_queue, frame_buffer) processes = [ artist, ShapeDetector(Thread, "detector", stop_process_event, log_queue, frame_queue, frame_buffer), Classifier(Thread, "classifier", stop_process_event, log_queue, subscriber_queue, frame_buffer, path, group, latch, kwargs={'log_level': DEBUG}) ] artist.subscribe(subscriber_queue) for process in processes: process.start() try: latch.wait() finally: stop_process_event.set() for process in processes: process.join(30) stop_logging_event.set() log_queue.join()
def _setup(self): self._processes = [] self._stop_events += [Event()] self._frame_queue = Queue() all_semaphores = {} for camera in self._config['cameras']: camera_name = next(iter(camera)) camera_config = camera[camera_name] buffer_size = 10 frame_buffer_in = FrameBuffer(buffer_size, camera_config['width'], camera_config['height']) frame_buffer_out = FrameBuffer(buffer_size, camera_config['width'], camera_config['height']) decoder_stop_event = Event() decoder_queue_semaphore = BoundedSemaphore(1) all_semaphores[camera_name] = decoder_queue_semaphore decoder_queue = BalancedQueue(self._frame_queue, {camera_name: decoder_queue_semaphore}, camera_name) decoder = FFmpegDecoder(camera_name, decoder_stop_event, self._log_queue, decoder_queue, frame_buffer_in, camera_config['ffmpeg']['decoder'], self._config_path, kwargs={'log_level': self._args.log_level}) self._processes.append(decoder) self._stop_events.append(decoder_stop_event) filters = self._create_filters(camera_config) detection_sieve_queue = Queue(1) detection_sieve = DetectionSieve(camera_name, self._stop_events[0], self._log_queue, detection_sieve_queue, frame_buffer_in, filters, decoder.rate_limiter, kwargs={'log_level': self._args.log_level}) self._processes.append(detection_sieve) decoder.subscribe(detection_sieve_queue) visual_effects_queue = Queue(1) visual_effects = VisualEffects(camera_name, self._stop_events[0], self._log_queue, visual_effects_queue, frame_buffer_in, frame_buffer_out, self._create_effects(camera_config), kwargs={'log_level': self._args.log_level}) self._processes.append(visual_effects) encoder, mpegts_reader, mpegts_buffer \ = self._create_encoder(camera_config, camera_name, frame_buffer_out, buffer_size, detection_sieve, visual_effects, visual_effects_queue) mqtt = self._create_mqtt(camera_config, camera_name, frame_buffer_in, decoder, decoder_stop_event, detection_sieve) snapshot_queue = Queue(1) snapshot = Snapshot(camera_name, self._stop_events[0], self._log_queue, snapshot_queue, frame_buffer_in, camera_config, self._create_effects(camera_config), kwargs={'topic': self.app_name, 'log_level': self._args.log_level}) self._processes.append(snapshot) detection_sieve.subscribe(snapshot_queue) self._cameras[camera_name] = Camera(frame_buffer_in, frame_buffer_out, decoder, encoder, detection_sieve, mqtt, snapshot, visual_effects, visual_effects_queue, MotionJpeg.create_buffer(buffer_size), mpegts_reader, mpegts_buffer) self._detectors += create_object_detectors(Process, self._stop_events[0], self._log_queue, BalancedQueue(self._frame_queue, all_semaphores), {n: c.frame_buffer_in for n, c in self._cameras.items()}, self._args.model_path, kwargs={'log_level': self._args.log_level}) self._processes += self._detectors
def test_subscribe(self): """Tests the coherence among the processes sharing the same frame buffer. The queues tying the processes are of limited size (1) and the workers are a bit slower, than the reader. The reader has to drop frames if any of the workers is busy handling previous frame. The frame state latch must get back to READY state, if neither the main worker nor subscriber can pick up the next frame. This ensures the buffer will never overflow. """ width = 500 height = 500 frame_buffer_in = FrameBuffer(10, width, height) frame_buffer_out = FrameBuffer(10, width, height) frame_queue = Queue(1) subscriber_queue = Queue(1) subscriber_queue1 = Queue(1) subscriber_queue2 = Queue(1) subscriber_queue3 = Queue(1) log_queue = Queue() getLogger().addHandler(QueueHandler(log_queue)) stop_process_event = Event() latch = CountDownLatch(100) effects = [CopyHeaderEffect(), CopyImageEffect(), DrawEffect()] artist = Artist("artist", stop_process_event, log_queue, frame_queue, frame_buffer_in) conductor = VisualEffects("conductor", stop_process_event, log_queue, subscriber_queue, frame_buffer_in, frame_buffer_out, effects) processes = [ artist, conductor, LogHandler(Thread, "logger", stop_process_event, log_queue, filename=None), ShapeDetector(Process, "detector", stop_process_event, log_queue, frame_queue, frame_buffer_in), ShapeCounter(Thread, "counter1", stop_process_event, log_queue, subscriber_queue1, frame_buffer_out, latch), ShapeCounter(Thread, "counter2", stop_process_event, log_queue, subscriber_queue2, frame_buffer_out, latch), ShapeCounter(Thread, "counter3", stop_process_event, log_queue, subscriber_queue3, frame_buffer_out, latch) ] artist.subscribe(subscriber_queue) conductor.subscribe(subscriber_queue1) conductor.subscribe(subscriber_queue2) conductor.subscribe(subscriber_queue3) for process in processes: process.start() try: self.assertTrue(latch.wait(15)) finally: stop_process_event.set() for process in processes: process.join(30) conductor.unsubscribe(subscriber_queue1) conductor.unsubscribe(subscriber_queue2) conductor.unsubscribe(subscriber_queue3) artist.unsubscribe(subscriber_queue)