def __init__(self, media_stream_descriptor): try: # media descriptor holding input data format self.media_stream_descriptor = media_stream_descriptor # Get how data will be transferred if (self.media_stream_descriptor.WhichOneof( "data_transfer_properties") is None): self.content_transfer_type = TransferType.BYTES elif self.media_stream_descriptor.HasField( "shared_memory_buffer_transfer_properties"): self.content_transfer_type = TransferType.REFERENCE elif self.media_stream_descriptor.HasField( "shared_memory_segments_transfer_properties"): self.content_transfer_type = TransferType.HANDLE # Setup if shared mem used if self.content_transfer_type == TransferType.REFERENCE: # Create shared memory accessor specific to the client self.shared_memory_manager = SharedMemoryManager( name=self.media_stream_descriptor. shared_memory_buffer_transfer_properties.handle_name, size=self.media_stream_descriptor. shared_memory_buffer_transfer_properties.length_bytes, ) else: self.shared_memory_manager = None except: log_exception(get_logger("State")) raise
def _generate_media_stream_message(self, gva_sample): message = json.loads(list(gva_sample.video_frame.messages())[0]) msg = extension_pb2.MediaStreamMessage() msg.ack_sequence_number = message["sequence_number"] msg.media_sample.timestamp = message["timestamp"] for region in gva_sample.video_frame.regions(): inference = msg.media_sample.inferences.add() attributes = [] obj_label = None obj_confidence = 0 obj_left = 0 obj_width = 0 obj_top = 0 obj_height = 0 for tensor in region.tensors(): name = tensor.name() if (name == 'detection'): obj_confidence = region.confidence() obj_label = region.label() obj_left, obj_top, obj_width, obj_height = region.normalized_rect( ) inference.type = inferencing_pb2.Inference.InferenceType.ENTITY # pylint: disable=no-member if region.object_id(): #Tracking obj_id = region.object_id() attributes.append(["object_id", str(obj_id), 0]) elif tensor["label"]: #Classification attr_name = name attr_label = tensor["label"] attr_confidence = region.confidence() attributes.append([attr_name, attr_label, attr_confidence]) if obj_label is not None: try: entity = inferencing_pb2.Entity( tag=inferencing_pb2.Tag(value=obj_label, confidence=obj_confidence), box=inferencing_pb2.Rectangle(l=obj_left, t=obj_top, w=obj_width, h=obj_height)) for attr in attributes: attribute = inferencing_pb2.Attribute( name=attr[0], value=attr[1], confidence=attr[2]) entity.attributes.append(attribute) except: log_exception(self._logger) raise inference.entity.CopyFrom(entity) return msg
def __init__(self, descriptor, shared_memory_manager, queue): try: self._request_seq_num = 1 self._descriptor = descriptor self._shared_memory_manager = shared_memory_manager self._queue = queue except: log_exception() raise
def _generate_gva_sample(self, client_state, request): new_sample = None try: # Get reference to raw bytes if client_state.content_transfer_type == TransferType.BYTES: raw_bytes = memoryview( request.media_sample.content_bytes.bytes) elif client_state.content_transfer_type == TransferType.REFERENCE: # Data sent over shared memory buffer address_offset = request.media_sample.content_reference.address_offset length_bytes = request.media_sample.content_reference.length_bytes # Get memory reference to (in readonly mode) data sent over shared memory raw_bytes = client_state.shared_memory_manager.read_bytes( address_offset, length_bytes) # Get encoding details of the media sent by client encoding = client_state.media_stream_descriptor.media_descriptor.video_frame_sample_format.encoding # Handle RAW content (Just place holder for the user to handle each variation...) if encoding == client_state.media_stream_descriptor.media_descriptor.video_frame_sample_format.Encoding.RAW: pixel_format = client_state.media_stream_descriptor.media_descriptor\ .video_frame_sample_format.pixel_format caps_format = None if pixel_format == media_pb2.VideoFrameSampleFormat.PixelFormat.RGBA: caps_format = 'RGBA' elif pixel_format == media_pb2.VideoFrameSampleFormat.PixelFormat.RGB24: caps_format = 'RGB' elif pixel_format == media_pb2.VideoFrameSampleFormat.PixelFormat.BGR24: caps_format = 'BGR' if caps_format is not None: caps = ''.join(("video/x-raw,format=", caps_format, ",width=", str(client_state.media_stream_descriptor.media_descriptor\ .video_frame_sample_format.dimensions.width), ",height=", str(client_state.media_stream_descriptor.media_descriptor\ .video_frame_sample_format.dimensions.height))) new_sample = GvaFrameData( bytes(raw_bytes), caps, message={ 'sequence_number': request.sequence_number, 'timestamp': request.media_sample.timestamp }) else: self._logger.info('Sample format is not RAW') except: log_exception(self._logger) raise return new_sample
def get_memory_slot(self, sequence_number, content_bytes): try: memory_slot = self._shared_memory_manager.get_empty_slot( sequence_number, len(content_bytes)) if memory_slot is None: return None self._shared_memory_manager.write_bytes( memory_slot[0], content_bytes) except Exception: log_exception() raise return memory_slot
def main(): try: args = parse_args() _log_options(args) frame_source = None frame_queue = queue.Queue() result_queue = queue.Queue() msp = MediaStreamProcessor(args.grpc_server_address, args.use_shared_memory) _, extension = os.path.splitext(args.sample_file) if extension in ['.png', '.jpg']: frame_source = ImageSource(args.sample_file, args.loop_count) elif extension in ['.mp4']: frame_source = VideoSource(args.sample_file) else: print("{}: unsupported file type".format(args.sample_file)) sys.exit(1) width, height = frame_source.dimensions() print("{} {}".format(width, height)) msp.start(width, height, frame_queue, result_queue) with open(args.output_file, "w") as output: image = frame_source.get_frame() while image: frame_queue.put(image) while not result_queue.empty(): result = result_queue.get() print_result(result, output) image = frame_source.get_frame() frame_queue.put(None) result = result_queue.get() while result: print_result(result, output) result = result_queue.get() frame_source.close() except Exception: log_exception() sys.exit(-1)
def get_media_stream_descriptor(self, width, height, extension_config): try: smbtp = None if self._shared_memory_manager: smbtp = extension_pb2.SharedMemoryBufferTransferProperties( handle_name=self._shared_memory_manager.shm_file_name, length_bytes=self._shared_memory_manager.shm_file_size, ) media_stream_descriptor = extension_pb2.MediaStreamDescriptor( graph_identifier=extension_pb2.GraphIdentifier( media_services_arm_id="", graph_instance_name="SampleGraph1", graph_node_name="SampleGraph1", ), extension_configuration=extension_config, media_descriptor=media_pb2.MediaDescriptor( timescale=90000, # pylint: disable=no-member # E1101: Class 'VideoFrameSampleFormat' has no 'Encoding' member (no-member) # E1101: Class 'VideoFrameSampleFormat' has no 'PixelFormat' member (no-member) video_frame_sample_format=media_pb2.VideoFrameSampleFormat( encoding=media_pb2.VideoFrameSampleFormat.Encoding. Value("RAW"), pixel_format=media_pb2.VideoFrameSampleFormat. PixelFormat.Value("BGR24"), dimensions=media_pb2.Dimensions( width=width, height=height, ), ), ), shared_memory_buffer_transfer_properties=smbtp, ) except Exception: log_exception() raise return media_stream_descriptor
def __init__(self, grpc_server_address, use_shared_memory, frame_queue_size, frame_size): try: # Full address including port number i.e. "localhost:44001" self._grpc_server_address = grpc_server_address self._shared_memory_manager = None if use_shared_memory: shared_memory_size = (frame_queue_size * frame_size if frame_queue_size else 100 * frame_size) self._shared_memory_manager = SharedMemoryManager( os.O_RDWR | os.O_SYNC | os.O_CREAT, name=None, size=shared_memory_size, ) self._grpc_channel = grpc.insecure_channel( self._grpc_server_address) self._grpc_stub = extension_pb2_grpc.MediaGraphExtensionStub( self._grpc_channel) except Exception: log_exception() raise
def ProcessMediaStream(self, requestIterator, context): requests_received = 0 responses_sent = 0 # First message from the client is (must be) MediaStreamDescriptor request = next(requestIterator) requests_received += 1 # Extract message IDs request_seq_num = request.sequence_number request_ack_seq_num = request.ack_sequence_number # State object per client client_state = State(request.media_stream_descriptor) self._logger.info("[Received] SeqNum: {0:07d} | " "AckNum: {1}\nMediaStreamDescriptor:\n{2}".format( request_seq_num, request_ack_seq_num, client_state.media_stream_descriptor, )) # First message response ... media_stream_message = extension_pb2.MediaStreamMessage( sequence_number=1, ack_sequence_number=request_seq_num, media_stream_descriptor=extension_pb2.MediaStreamDescriptor( media_descriptor=media_pb2.MediaDescriptor( timescale=client_state.media_stream_descriptor. media_descriptor.timescale)), ) responses_sent += 1 yield media_stream_message final_pipeline_parameters = {} if self._version.startswith("debug"): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") location = os.path.join(tempfile.gettempdir(), "vaserving", self._version, timestamp) os.makedirs(os.path.abspath(location)) final_pipeline_parameters = { "location": os.path.join(location, "frame_%07d.jpeg") } try: if self._pipeline_parameter_arg: pipeline_parameters = {} if os.path.isfile(self._pipeline_parameter_arg): with open(self._pipeline_parameter_arg) as json_file: pipeline_parameters = json.load(json_file) else: pipeline_parameters = json.loads( self._pipeline_parameter_arg) final_pipeline_parameters.update(pipeline_parameters) except ValueError: self._logger.error("Issue loading json parameters: {}".format( self._pipeline_parameter_arg)) raise self._logger.info("Pipeline Name : {}".format(self._pipeline)) self._logger.info("Pipeline Version : {}".format(self._version)) self._logger.info( "Pipeline Parameters : {}".format(final_pipeline_parameters)) detect_input = Queue(maxsize=self._input_queue_size) detect_output = Queue() # Start object detection pipeline # It will wait until it receives frames via the detect_input queue detect_pipeline = VAServing.pipeline(self._pipeline, self._version) detect_pipeline.start( source={ "type": "application", "class": "GStreamerAppSource", "input": detect_input, "mode": "push", }, destination={ "type": "application", "class": "GStreamerAppDestination", "output": detect_output, "mode": "frames", }, parameters=final_pipeline_parameters, ) # Process rest of the MediaStream message sequence for request in requestIterator: try: if requests_received - responses_sent >= self._input_queue_size: queued_samples = self._get_queued_samples(detect_output, block=True) else: queued_samples = [] # Read request id, sent by client request_seq_num = request.sequence_number self._logger.info( "[Received] SeqNum: {0:07d}".format(request_seq_num)) requests_received += 1 gva_sample = self._generate_gva_sample(client_state, request) detect_input.put(gva_sample) queued_samples.extend(self._get_queued_samples(detect_output)) if context.is_active(): for gva_sample in queued_samples: if gva_sample: media_stream_message = self._generate_media_stream_message( gva_sample) responses_sent += 1 self._logger.info( "[Sent] AckSeqNum: {0:07d}".format( media_stream_message.ack_sequence_number)) yield media_stream_message else: break if detect_pipeline.status().state.stopped(): break except: log_exception(self._logger) raise if detect_pipeline.status().state.stopped(): try: raise Exception(detect_pipeline.status().state) except: log_exception(self._logger) raise # After the server has finished processing all the request iterator objects # Push a None object into the input queue. # When the None object comes out of the output queue, we know we've finished # processing all requests gva_sample = None if not detect_pipeline.status().state.stopped(): detect_input.put(None) gva_sample = detect_output.get() while gva_sample: media_stream_message = self._generate_media_stream_message( gva_sample) responses_sent += 1 self._logger.info("[Sent] AckSeqNum: {0:07d}".format( media_stream_message.ack_sequence_number)) if context.is_active(): yield media_stream_message gva_sample = detect_output.get() self._logger.info( "Done processing messages: Received: {}, Sent: {}".format( requests_received, responses_sent)) self._logger.info("MediaStreamDescriptor:\n{0}".format( client_state.media_stream_descriptor))
def main(): try: args = parse_args() _log_options(args) frame_delay = 1 / args.frame_rate if args.frame_rate > 0 else 0 frame_source = None frame_queue = queue.Queue(args.frame_queue_size) result_queue = queue.Queue() frames_sent = 0 frames_received = 0 prev_fps_delta = 0 start_time = None frame_source = VideoSource(args.sample_file, args.loop_count) width, height = frame_source.dimensions() image = frame_source.get_frame() if not image: logging.error( "Error getting frame from video source: {}".format(args.sample_file) ) sys.exit(1) msp = MediaStreamProcessor( args.grpc_server_address, args.use_shared_memory, args.frame_queue_size, len(image), ) extension_config = json.dumps(create_extension_config(args)) msp.start(width, height, frame_queue, result_queue, extension_config) with open(args.output_file, "w") as output: start_time = time.time() result = True while image and result: frame_queue.put(image) while not result_queue.empty(): result = result_queue.get() if isinstance(result, Exception): logging.error(result) frame_source.close() sys.exit(1) frames_received += 1 prev_fps_delta = _log_fps( start_time, frames_received, prev_fps_delta, args.fps_interval ) _log_result(result, output) image = frame_source.get_frame() time.sleep(frame_delay) frames_sent += 1 if result: frame_queue.put(None) result = result_queue.get() while result: if isinstance(result, Exception): logging.error(result) frame_source.close() sys.exit(1) frames_received += 1 prev_fps_delta = _log_fps( start_time, frames_received, prev_fps_delta, args.fps_interval ) _log_result(result, output) result = result_queue.get() frame_source.close() delta = time.time() - start_time logging.info( "Start Time: {} End Time: {} Frames: Tx {} Rx {} FPS: {}".format( start_time, start_time + delta, frames_sent, frames_received, (frames_received / delta) if delta > 0 else None, ) ) except Exception: log_exception() return -1 if frames_sent != frames_received: logging.error("Sent {} requests, received {} responses".format( frames_sent, frames_received )) return 1 return 0