def __init__(self, filename, file=None): if file is not None: raise NotImplementedError('TODO: Load from file stream') self._file = av.avbin_open_filename(filename) if not self._file: raise AVbinException('Could not open "%s"' % filename) self._video_stream = None self._video_stream_index = -1 self._audio_stream = None self._audio_stream_index = -1 file_info = AVbinFileInfo() file_info.structure_size = ctypes.sizeof(file_info) av.avbin_file_info(self._file, ctypes.byref(file_info)) self._duration = timestamp_from_avbin(file_info.duration) # Pick the first video and audio streams found, ignore others. for i in range(file_info.n_streams): info = AVbinStreamInfo() info.structure_size = ctypes.sizeof(info) av.avbin_stream_info(self._file, i, info) if (info.type == AVBIN_STREAM_TYPE_VIDEO and not self._video_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.video_format = VideoFormat(width=info.u.video.width, height=info.u.video.height) if info.u.video.sample_aspect_num != 0: self.video_format.sample_aspect = ( float(info.u.video.sample_aspect_num) / info.u.video.sample_aspect_den) self._video_stream = stream self._video_stream_index = i elif (info.type == AVBIN_STREAM_TYPE_AUDIO and info.u.audio.sample_bits in (8, 16) and info.u.audio.channels in (1, 2) and not self._audio_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.audio_format = AudioFormat( channels=info.u.audio.channels, sample_size=info.u.audio.sample_bits, sample_rate=info.u.audio.sample_rate) self._audio_stream = stream self._audio_stream_index = i self._packet = AVbinPacket() self._packet.structure_size = ctypes.sizeof(self._packet) self._packet.stream_index = -1 self._events = list() self._video_images = BufferedImageQueue() # Timestamp of last video packet added to decoder queue. self._video_timestamp = 0 if self.audio_format: self._audio_buffer = \ (ctypes.c_uint8 * av.avbin_get_audio_buffer_size())() self._buffered_audio_data = list() if self.video_format: self._decode_thread = WorkerThread() self._decode_thread.start() self._requested_video_frame_id = -1 self._lock = threading.Lock()
def __init__(self, filename, file=None): if file is not None: raise NotImplementedError('TODO: Load from file stream') self._file = av.avbin_open_filename(filename) if not self._file: raise AVbinException('Could not open "%s"' % filename) self._video_stream = None self._video_stream_index = -1 self._audio_stream = None self._audio_stream_index = -1 file_info = AVbinFileInfo() file_info.structure_size = ctypes.sizeof(file_info) av.avbin_file_info(self._file, ctypes.byref(file_info)) self._duration = timestamp_from_avbin(file_info.duration) # Pick the first video and audio streams found, ignore others. for i in range(file_info.n_streams): info = AVbinStreamInfo() info.structure_size = ctypes.sizeof(info) av.avbin_stream_info(self._file, i, info) if (info.type == AVBIN_STREAM_TYPE_VIDEO and not self._video_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.video_format = VideoFormat( width=info.u.video.width, height=info.u.video.height) if info.u.video.sample_aspect_num != 0: self.video_format.sample_aspect = ( float(info.u.video.sample_aspect_num) / info.u.video.sample_aspect_den) self._video_stream = stream self._video_stream_index = i elif (info.type == AVBIN_STREAM_TYPE_AUDIO and info.u.audio.sample_bits in (8, 16) and info.u.audio.channels in (1, 2) and not self._audio_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.audio_format = AudioFormat( channels=info.u.audio.channels, sample_size=info.u.audio.sample_bits, sample_rate=info.u.audio.sample_rate) self._audio_stream = stream self._audio_stream_index = i self._packet = AVbinPacket() self._packet.structure_size = ctypes.sizeof(self._packet) self._packet.stream_index = -1 self._events = list() self._video_images = BufferedImageQueue() # Timestamp of last video packet added to decoder queue. self._video_timestamp = 0 if self.audio_format: self._audio_buffer = \ (ctypes.c_uint8 * av.avbin_get_audio_buffer_size())() self._buffered_audio_data = list() if self.video_format: self._decode_thread = WorkerThread() self._decode_thread.start() self._requested_video_frame_id = -1 self._lock = threading.Lock()
class AVbinSource(StreamingSource): def __init__(self, filename, file=None): if file is not None: raise NotImplementedError('TODO: Load from file stream') self._file = av.avbin_open_filename(filename) if not self._file: raise AVbinException('Could not open "%s"' % filename) self._video_stream = None self._video_stream_index = -1 self._audio_stream = None self._audio_stream_index = -1 file_info = AVbinFileInfo() file_info.structure_size = ctypes.sizeof(file_info) av.avbin_file_info(self._file, ctypes.byref(file_info)) self._duration = timestamp_from_avbin(file_info.duration) # Pick the first video and audio streams found, ignore others. for i in range(file_info.n_streams): info = AVbinStreamInfo() info.structure_size = ctypes.sizeof(info) av.avbin_stream_info(self._file, i, info) if (info.type == AVBIN_STREAM_TYPE_VIDEO and not self._video_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.video_format = VideoFormat(width=info.u.video.width, height=info.u.video.height) if info.u.video.sample_aspect_num != 0: self.video_format.sample_aspect = ( float(info.u.video.sample_aspect_num) / info.u.video.sample_aspect_den) self._video_stream = stream self._video_stream_index = i elif (info.type == AVBIN_STREAM_TYPE_AUDIO and info.u.audio.sample_bits in (8, 16) and info.u.audio.channels in (1, 2) and not self._audio_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.audio_format = AudioFormat( channels=info.u.audio.channels, sample_size=info.u.audio.sample_bits, sample_rate=info.u.audio.sample_rate) self._audio_stream = stream self._audio_stream_index = i self._packet = AVbinPacket() self._packet.structure_size = ctypes.sizeof(self._packet) self._packet.stream_index = -1 self._events = list() self._video_images = BufferedImageQueue() # Timestamp of last video packet added to decoder queue. self._video_timestamp = 0 if self.audio_format: self._audio_buffer = \ (ctypes.c_uint8 * av.avbin_get_audio_buffer_size())() self._buffered_audio_data = list() if self.video_format: self._decode_thread = WorkerThread() self._decode_thread.start() self._requested_video_frame_id = -1 self._lock = threading.Lock() def __del__(self): if _debug: print('del avbin source') try: if self._video_stream: av.avbin_close_stream(self._video_stream) if self._audio_stream: av.avbin_close_stream(self._audio_stream) av.avbin_close_file(self._file) except: pass # TODO: TODO call this / add to source api def delete(self): if self.video_format: self._decode_thread.stop() def seek(self, timestamp): if _debug: print('AVbin seek', timestamp) av.avbin_seek_file(self._file, timestamp_to_avbin(timestamp)) self._audio_packet_size = 0 del self._events[:] del self._buffered_audio_data[:] if self.video_format: self._video_timestamp = 0 self._video_images.clear() self._lock.acquire() self._requested_video_frame_id = -1 self._lock.release() self._decode_thread.clear_jobs() def _queue_video_frame(self): # Add the next video frame to the decode queue (may return without # adding anything to the queue if eos) if _debug: print('_single_video_frame') while True: if not self._get_packet(): break packet_type, packet = self._process_packet() if packet_type == 'video': return packet.id def _get_packet(self): # Read a packet into self._packet. Returns True if OK, False if no # more packets are in stream. return av.avbin_read(self._file, self._packet) == AVBIN_RESULT_OK def _process_packet(self): # Returns (packet_type, packet), where packet_type = 'video' or # 'audio'; and packet is VideoPacket or AudioData. In either case, # packet is buffered or queued for decoding; no further action is # necessary. Returns (None, None) if packet was neither type. if self._packet.stream_index == self._video_stream_index: if self._packet.timestamp < 0: # TODO: TODO # AVbin needs hack to decode timestamp for B frames in # some containers (OGG?). See # http://www.dranger.com/ffmpeg/tutorial05.html # For now we just drop these frames. return None, None video_packet = VideoPacket(self._packet) if _debug: print('Created and queued frame %d (%f)' % (video_packet.id, video_packet.timestamp)) self._events.append( MediaEvent(video_packet.timestamp, 'on_video_frame', video_packet.id)) self._video_timestamp = max(self._video_timestamp, video_packet.timestamp) self._decode_thread.put_job( lambda: self._decode_video_frame(video_packet)) return 'video', video_packet elif self._packet.stream_index == self._audio_stream_index: audio_data = self._decode_audio_packet() if _debug: print('Got an audio packet at', audio_data.timestamp) if audio_data: self._buffered_audio_data.append(audio_data) return 'audio', audio_data return None, None def _get_audio_data(self, bytes): try: audio_data = self._buffered_audio_data.pop(0) audio_data_timeend = audio_data.timestamp + audio_data.duration except IndexError: audio_data = None audio_data_timeend = self._video_timestamp + 1 if _debug: print('_get_audio_data') have_video_work = False # Keep reading packets until we have an audio packet and all the # associated video packets have been enqueued on the decoder thread. while not audio_data or (self._video_stream and self._video_timestamp < audio_data_timeend): if not self._get_packet(): break packet_type, packet = self._process_packet() if packet_type == 'video': have_video_work = True elif not audio_data and packet_type == 'audio': audio_data = self._buffered_audio_data.pop(0) if _debug: print('Got requested audio packet at', audio_data.timestamp) audio_data_timeend = audio_data.timestamp + audio_data.duration if have_video_work: # Give decoder thread a chance to run before we return this audio # data. time.sleep(0) if not audio_data: if _debug: print('_get_audio_data returning None') return None while self._events and self._events[0].timestamp <= audio_data_timeend: event = self._events.pop(0) if event.timestamp >= audio_data.timestamp: event.timestamp -= audio_data.timestamp audio_data.events.append(event) if _debug: print( '_get_audio_data returning ts %f with events' % audio_data.timestamp, audio_data.events) print('remaining events are', self._events) return audio_data def _decode_audio_packet(self): packet = self._packet size_out = ctypes.c_int(len(self._audio_buffer)) while True: audio_packet_ptr = ctypes.cast(packet.data, ctypes.c_void_p) audio_packet_size = packet.size used = av.avbin_decode_audio(self._audio_stream, audio_packet_ptr, audio_packet_size, self._audio_buffer, size_out) if used < 0: self._audio_packet_size = 0 break audio_packet_ptr.value += used audio_packet_size -= used if size_out.value <= 0: continue buffer = ctypes.string_at(self._audio_buffer, size_out) duration = float(len(buffer)) / self.audio_format.bytes_per_second self._audio_packet_timestamp = \ timestamp = timestamp_from_avbin(packet.timestamp) return AudioData(buffer, len(buffer), timestamp, duration, list()) def _decode_video_packet(self, packet): timestamp = packet.timestamp # TODO: unused width = self.video_format.width height = self.video_format.height pitch = width * 3 buffer = (ctypes.c_uint8 * (pitch * height))() result = av.avbin_decode_video(self._video_stream, packet.data, packet.size, buffer) if result < 0: image_data = None else: image_data = image.ImageData(width, height, 'RGB', buffer, pitch) return BufferedImage(image_data, packet.id) def get_next_video_timestamp(self): raise TODO if not self.video_format: return try: img = self._buffered_images[0] except IndexError: img = self._next_image() self._buffered_images.append(img) if img: return img.timestamp def get_next_video_frame(self): # See caveat below regarding get_next_video_frame_id id = self.get_next_video_frame_id() return self.get_video_frame(id) def get_next_video_frame_id(self): id = self._video_images.get_first_id() if id is not None: return id # Nothing already decoded, let's queue something now and return the # id. (Not perfect, because something might be queued... but we're # handling the common case -- seeking) id = self._queue_video_frame() return id def _decode_video_frame(self, packet): if _debug: print('decoding video frame', packet.id) buffered_image = self._decode_video_packet(packet) self._lock.acquire() requested_video_frame_id = self._requested_video_frame_id self._lock.release() if packet.id >= requested_video_frame_id: self._video_images.put(buffered_image) if _debug: print('done decoding video frame', packet.id) def get_video_frame(self, id): if _debug: print('get_video_frame', id) if not self.video_format: return if id <= self._requested_video_frame_id: return self._lock.acquire() self._requested_video_frame_id = id self._lock.release() buffered_image = self._video_images.get(id) if _debug: print('get_video_frame(%r) -> %r' % (id, buffered_image)) return buffered_image.image
class AVbinSource(StreamingSource): def __init__(self, filename, file=None): if file is not None: raise NotImplementedError('TODO: Load from file stream') self._file = av.avbin_open_filename(filename) if not self._file: raise AVbinException('Could not open "%s"' % filename) self._video_stream = None self._video_stream_index = -1 self._audio_stream = None self._audio_stream_index = -1 file_info = AVbinFileInfo() file_info.structure_size = ctypes.sizeof(file_info) av.avbin_file_info(self._file, ctypes.byref(file_info)) self._duration = timestamp_from_avbin(file_info.duration) # Pick the first video and audio streams found, ignore others. for i in range(file_info.n_streams): info = AVbinStreamInfo() info.structure_size = ctypes.sizeof(info) av.avbin_stream_info(self._file, i, info) if (info.type == AVBIN_STREAM_TYPE_VIDEO and not self._video_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.video_format = VideoFormat( width=info.u.video.width, height=info.u.video.height) if info.u.video.sample_aspect_num != 0: self.video_format.sample_aspect = ( float(info.u.video.sample_aspect_num) / info.u.video.sample_aspect_den) self._video_stream = stream self._video_stream_index = i elif (info.type == AVBIN_STREAM_TYPE_AUDIO and info.u.audio.sample_bits in (8, 16) and info.u.audio.channels in (1, 2) and not self._audio_stream): stream = av.avbin_open_stream(self._file, i) if not stream: continue self.audio_format = AudioFormat( channels=info.u.audio.channels, sample_size=info.u.audio.sample_bits, sample_rate=info.u.audio.sample_rate) self._audio_stream = stream self._audio_stream_index = i self._packet = AVbinPacket() self._packet.structure_size = ctypes.sizeof(self._packet) self._packet.stream_index = -1 self._events = list() self._video_images = BufferedImageQueue() # Timestamp of last video packet added to decoder queue. self._video_timestamp = 0 if self.audio_format: self._audio_buffer = \ (ctypes.c_uint8 * av.avbin_get_audio_buffer_size())() self._buffered_audio_data = list() if self.video_format: self._decode_thread = WorkerThread() self._decode_thread.start() self._requested_video_frame_id = -1 self._lock = threading.Lock() def __del__(self): if _debug: print('del avbin source') try: if self._video_stream: av.avbin_close_stream(self._video_stream) if self._audio_stream: av.avbin_close_stream(self._audio_stream) av.avbin_close_file(self._file) except: pass # TODO: TODO call this / add to source api def delete(self): if self.video_format: self._decode_thread.stop() def seek(self, timestamp): if _debug: print('AVbin seek', timestamp) av.avbin_seek_file(self._file, timestamp_to_avbin(timestamp)) self._audio_packet_size = 0 del self._events[:] del self._buffered_audio_data[:] if self.video_format: self._video_timestamp = 0 self._video_images.clear() self._lock.acquire() self._requested_video_frame_id = -1 self._lock.release() self._decode_thread.clear_jobs() def _queue_video_frame(self): # Add the next video frame to the decode queue (may return without # adding anything to the queue if eos) if _debug: print('_single_video_frame') while True: if not self._get_packet(): break packet_type, packet = self._process_packet() if packet_type == 'video': return packet.id def _get_packet(self): # Read a packet into self._packet. Returns True if OK, False if no # more packets are in stream. return av.avbin_read(self._file, self._packet) == AVBIN_RESULT_OK def _process_packet(self): # Returns (packet_type, packet), where packet_type = 'video' or # 'audio'; and packet is VideoPacket or AudioData. In either case, # packet is buffered or queued for decoding; no further action is # necessary. Returns (None, None) if packet was neither type. if self._packet.stream_index == self._video_stream_index: if self._packet.timestamp < 0: # TODO: TODO # AVbin needs hack to decode timestamp for B frames in # some containers (OGG?). See # http://www.dranger.com/ffmpeg/tutorial05.html # For now we just drop these frames. return None, None video_packet = VideoPacket(self._packet) if _debug: print('Created and queued frame %d (%f)' % (video_packet.id, video_packet.timestamp)) self._events.append(MediaEvent(video_packet.timestamp, 'on_video_frame', video_packet.id)) self._video_timestamp = max(self._video_timestamp, video_packet.timestamp) self._decode_thread.put_job( lambda: self._decode_video_frame(video_packet)) return 'video', video_packet elif self._packet.stream_index == self._audio_stream_index: audio_data = self._decode_audio_packet() if _debug: print('Got an audio packet at', audio_data.timestamp) if audio_data: self._buffered_audio_data.append(audio_data) return 'audio', audio_data return None, None def _get_audio_data(self, bytes): try: audio_data = self._buffered_audio_data.pop(0) audio_data_timeend = audio_data.timestamp + audio_data.duration except IndexError: audio_data = None audio_data_timeend = self._video_timestamp + 1 if _debug: print('_get_audio_data') have_video_work = False # Keep reading packets until we have an audio packet and all the # associated video packets have been enqueued on the decoder thread. while not audio_data or ( self._video_stream and self._video_timestamp < audio_data_timeend): if not self._get_packet(): break packet_type, packet = self._process_packet() if packet_type == 'video': have_video_work = True elif not audio_data and packet_type == 'audio': audio_data = self._buffered_audio_data.pop(0) if _debug: print( 'Got requested audio packet at', audio_data.timestamp) audio_data_timeend = audio_data.timestamp + audio_data.duration if have_video_work: # Give decoder thread a chance to run before we return this audio # data. time.sleep(0) if not audio_data: if _debug: print('_get_audio_data returning None') return None while self._events and self._events[0].timestamp <= audio_data_timeend: event = self._events.pop(0) if event.timestamp >= audio_data.timestamp: event.timestamp -= audio_data.timestamp audio_data.events.append(event) if _debug: print('_get_audio_data returning ts %f with events' % audio_data.timestamp, audio_data.events) print('remaining events are', self._events) return audio_data def _decode_audio_packet(self): packet = self._packet size_out = ctypes.c_int(len(self._audio_buffer)) while True: audio_packet_ptr = ctypes.cast(packet.data, ctypes.c_void_p) audio_packet_size = packet.size used = av.avbin_decode_audio(self._audio_stream, audio_packet_ptr, audio_packet_size, self._audio_buffer, size_out) if used < 0: self._audio_packet_size = 0 break audio_packet_ptr.value += used audio_packet_size -= used if size_out.value <= 0: continue buffer = ctypes.string_at(self._audio_buffer, size_out) duration = float(len(buffer)) / self.audio_format.bytes_per_second self._audio_packet_timestamp = \ timestamp = timestamp_from_avbin(packet.timestamp) return AudioData(buffer, len(buffer), timestamp, duration, list()) def _decode_video_packet(self, packet): timestamp = packet.timestamp # TODO: unused width = self.video_format.width height = self.video_format.height pitch = width * 3 buffer = (ctypes.c_uint8 * (pitch * height))() result = av.avbin_decode_video(self._video_stream, packet.data, packet.size, buffer) if result < 0: image_data = None else: image_data = image.ImageData(width, height, 'RGB', buffer, pitch) return BufferedImage(image_data, packet.id) def get_next_video_timestamp(self): raise TODO if not self.video_format: return try: img = self._buffered_images[0] except IndexError: img = self._next_image() self._buffered_images.append(img) if img: return img.timestamp def get_next_video_frame(self): # See caveat below regarding get_next_video_frame_id id = self.get_next_video_frame_id() return self.get_video_frame(id) def get_next_video_frame_id(self): id = self._video_images.get_first_id() if id is not None: return id # Nothing already decoded, let's queue something now and return the # id. (Not perfect, because something might be queued... but we're # handling the common case -- seeking) id = self._queue_video_frame() return id def _decode_video_frame(self, packet): if _debug: print('decoding video frame', packet.id) buffered_image = self._decode_video_packet(packet) self._lock.acquire() requested_video_frame_id = self._requested_video_frame_id self._lock.release() if packet.id >= requested_video_frame_id: self._video_images.put(buffered_image) if _debug: print('done decoding video frame', packet.id) def get_video_frame(self, id): if _debug: print('get_video_frame', id) if not self.video_format: return if id <= self._requested_video_frame_id: return self._lock.acquire() self._requested_video_frame_id = id self._lock.release() buffered_image = self._video_images.get(id) if _debug: print('get_video_frame(%r) -> %r' % (id, buffered_image)) return buffered_image.image