def __init__( self, buffer, start_offset, bytes_reverse=False, bits_reverse=False, output_reverse=False, bytes_to_cache=1 ): """Create a BitReader instance. buffer Source byte string to read from. start_offset Position in the block to start reading from. bytes_reverse If enabled, fetch successive bytes from the source in reverse order. bits_reverse If enabled, fetch bits starting from the most-significant bit (i.e. 0x80) through least-significant bit (0x01). output_reverse If enabled, return fetched bits starting from the most-significant bit (e.g. 0x80) through least-significant bit (0x01). bytes_to_cache Number of bytes to cache. Defaults to 1. Only useful for algorithms which change the position pointer mid-read. """ assert is_bytes( buffer ) assert start_offset in range( len( buffer ) ) self.buffer = buffer self.bits_reverse = bits_reverse self.bytes_reverse = bytes_reverse self.output_reverse = output_reverse self.pos = start_offset self.bytes_to_cache = bytes_to_cache self._fill_buffer()
def __init__(self, source_data=None, parent=None, preload_attrs=None, endian=None, cache_bytes=False): """Base class for Blocks. source_data Source data to construct Block with. Can be a byte string, dictionary of attribute: value pairs, or another Block object. parent Parent Block object where this Block is defined. Used for e.g. evaluating Refs. preload_attrs Attributes on the Block to set before importing the data. Used for linking in dependencies before loading. endian Platform endianness to use when interpreting the Block data. Useful for Blocks which have the same data layout but different endianness for stored numbers. Has no effect on fields with an predefined endianness. cache_bytes Cache the bytes equivalent of the Block. Useful for debugging the loading procedure. Defaults to False. """ self._field_data = {} self._ref_cache = {} if parent is not None: assert isinstance(parent, Block) self._parent = parent self._endian = endian if endian else ( parent._endian if parent else self._endian) if cache_bytes: self._cache_bytes = True if source_data and common.is_bytes(source_data): self._bytes = bytes(source_data) if preload_attrs: for attr, value in preload_attrs.items(): setattr(self, attr, value) # start the initial load of data if isinstance(source_data, Block): self.clone_data(source_data) elif isinstance(source_data, dict): # preload defaults, then overwrite with dictionary values self.import_data(None) self.update_data(source_data) else: self.import_data(source_data) # cache all refs for key, ref in self._refs.items(): ref.cache(self)
def histdump_iter(source, start=None, end=None, length=None, samples=0x10000, width=64, address_base=None): assert is_bytes(source) start, end = bounds(start, end, length, len(source)) start = max(start, 0) end = min(end, len(source)) if len(source) == 0 or (start == end == 0): return address_base_offset = address_base - start if address_base is not None else 0 for offset in range(start, end, samples): yield ansi.format_histdump_line( source, offset, length=samples, end=end, width=width, address_base_offset=address_base_offset) return
def pixdump_iter(source, start=None, end=None, length=None, width=64, height=None, palette=None): """Return the contents of a byte string as a 256 colour image. source The byte string to print. start Start offset to read from (default: start) end End offset to stop reading at (default: end) length Length to read in (optional replacement for end) width Width of image to render in pixels (default: 64) height Height of image to render in pixels (default: auto) palette List of Colours to use (default: test palette) """ assert is_bytes(source) if not palette: palette = colour.TEST_PALETTE start = 0 if (start is None) else start if (end is not None) and (length is not None): raise ValueError('Can\'t define both an end and a length!') elif (length is not None): end = start + length elif (end is not None): pass else: end = len(source) start = max(start, 0) end = min(end, len(source)) if len(source) == 0 or (start == end == 0): return iter(()) if height is None: height = math.ceil((end - start) / width) def data_fetch(x_pos, y_pos, frame): index = y_pos * width + x_pos + start if index >= end: return (0, 0, 0, 0) return palette[source[index]] return ansi.format_image_iter(data_fetch, width=width, height=height)
def __init__(self, source, start=None, end=None, length=None, major_len=8, minor_len=4, colour=True, address_base=None, before=2, after=2, title=None): assert is_bytes(source) self.source = source self.start, self.end = bounds(start, end, length, len(source)) if len(source) == 0 or (start == end == 0): return self.address_base_offset = address_base - start if address_base is not None else 0 self.major_len = major_len self.minor_len = minor_len self.colour = colour self.before = before self.after = after self.title = title self.stride = minor_len * major_len self.lines = [] self.last_printed = -1 self.output_buffer = {} self.printed = False
def normalise_audio_iter( source, format_type, field_size, signedness, endian, start=None, end=None, length=None, overlap=0, chunk_size=NORMALIZE_BUFFER ): assert is_bytes( source ) start, end = bounds( start, end, length, len( source ) ) increment = chunk_size+overlap for i in range( start, end, chunk_size ): yield normalise_audio( source, format_type, field_size, signedness, endian, start=i, end=None, length=increment )
def import_data(self, raw_buffer): """Import data from a byte array. raw_buffer Byte array to import from. """ klass = self.__class__ if raw_buffer is not None: assert common.is_bytes(raw_buffer) # raw_buffer = memoryview( raw_buffer ) self._field_data = {} logger.debug('{}: loading fields'.format(self)) for name in klass._fields: if raw_buffer is not None: if logger.isEnabledFor(logging.DEBUG): logger.debug('{} [{}]: input buffer'.format( name, klass._fields[name])) self._field_data[name] = klass._fields[name].get_from_buffer( raw_buffer, parent=self) if logger.isEnabledFor(logging.DEBUG): logger.debug('Result for {} [{}]: {}'.format( name, klass._fields[name], self._field_data[name])) else: self._field_data[name] = klass._fields[name].default if raw_buffer is not None: for name, check in klass._checks.items(): check.check_buffer(raw_buffer, parent=self) # if we have debug logging on, check the roundtrip works if logger.isEnabledFor(logging.INFO): test = self.export_data() if logger.getEffectiveLevel() <= logging.DEBUG: logger.debug('Stats for {}:'.format(self)) logger.debug('Import buffer size: {}'.format( len(raw_buffer))) logger.debug('Export size: {}'.format(len(test))) if test == raw_buffer: logger.debug('Content: exact match!') elif test == raw_buffer[:len(test)]: logger.debug('Content: exact match with overflow!') else: logger.debug('Content: different!') for x in utils.hexdump_diff_iter( raw_buffer[:len(test)], test): logger.debug(x) elif test != raw_buffer[:len(test)]: logger.info( '{} export produced changed output from import'.format( self)) # if raw_buffer: # raw_buffer.release() return
def resample_audio_iter(source, format_type, field_size, signedness, endian, channels, sample_rate, start=None, end=None, length=None, interpolation=AudioInterpolation.LINEAR, output_rate=RESAMPLE_RATE): if sample_rate == 0: yield 0.0 return assert is_bytes(source) start, end = bounds(start, end, length, len(source)) mixer = mix_linear if interpolation == AudioInterpolation.STEP: mixer = mix_step new_len = (end - start) * output_rate // sample_rate src_inc = NORMALIZE_BUFFER chunk_size = src_inc * channels src_iter = normalise_audio_iter(source, format_type, field_size, signedness, endian, start, end, overlap=0, chunk_size=chunk_size) src = next(src_iter, None) src_bound = src_inc for index_base in range(0, new_len): tgt_pos = index_base src_pos = sample_rate * tgt_pos / output_rate samp_index = math.floor(src_pos) % src_inc alpha = math.fmod(src_pos, 1.0) if src_pos > src_bound: src = next(src_iter, None) src_bound += src_inc if src is None: break a = 0.0 if samp_index >= len(src) else src[samp_index] b = 0.0 if samp_index + channels >= len(src) else src[samp_index + channels] yield mixer(a, b, alpha)
def pixdump_iter( source, start=None, end=None, length=None, width=64, height=None, palette=None ): """Return the contents of a byte string as a 256 colour image. source The byte string to print. start Start offset to read from (default: start) end End offset to stop reading at (default: end) length Length to read in (optional replacement for end) width Width of image to render in pixels (default: 64) height Height of image to render in pixels (default: auto) palette List of Colours to use (default: test palette) """ assert is_bytes( source ) if not palette: palette = colour.TEST_PALETTE start = 0 if (start is None) else start if (end is not None) and (length is not None): raise ValueError( 'Can\'t define both an end and a length!' ) elif (length is not None): end = start+length elif (end is not None): pass else: end = len( source ) start = max( start, 0 ) end = min( end, len( source ) ) if len( source ) == 0 or (start == end == 0): return iter(()) if height is None: height = math.ceil( (end-start)/width ) def data_fetch( x_pos, y_pos, frame ): index = y_pos*width + x_pos + start if index >= end: return (0, 0, 0, 0) return palette[source[index]] return ansi.format_image_iter( data_fetch, width=width, height=height )
def hexdump_iter(source, start=None, end=None, length=None, major_len=8, minor_len=4, colour=True, address_base=None): """Return the contents of a byte string in tabular hexadecimal/ASCII format. source The byte string to print. start Start offset to read from (default: start) end End offset to stop reading at (default: end) length Length to read in (optional replacement for end) major_len Number of hexadecimal groups per line minor_len Number of bytes per hexadecimal group colour Add ANSI colour formatting to output (default: true) address_base Base address to use for labels (default: start) Raises ValueError if both end and length are defined. """ assert is_bytes(source) start, end = bounds(start, end, length, len(source)) start = max(start, 0) end = min(end, len(source)) if len(source) == 0 or (start == end == 0): return address_base_offset = address_base - start if address_base is not None else 0 for offset in range(start, end, minor_len * major_len): yield ansi.format_hexdump_line(source, offset, end, major_len, minor_len, colour, address_base_offset=address_base_offset) return
def __init__(self, buffer=None, start_offset=None, bytes_reverse=False, bit_endian='big', io_endian='big'): """Create a BitStream instance. buffer Target byte array to read/write from. Defaults to an empty array. start_offset Position in the target to start reading from. Can be an integer byte offset, or a tuple containing the byte and bit offsets. Defaults to the start of the stream, depending on the endianness and ordering options. bytes_reverse If enabled, fetch successive bytes from the source in reverse order. bit_endian Endianness of the backing storage; either 'big' or 'little'. Defaults to big (i.e. starting from the most-significant bit (0x80) through least-significant bit (0x10)). io_endian Endianness of data returned from read/write; either 'big' or 'little'. Defaults to big (i.e. starting from the most-significant bit (0x80) through least-significant bit (0x10)). """ if buffer is None: self.buffer = bytearray() else: assert is_bytes(buffer) self.buffer = buffer self.bytes_reverse = bytes_reverse if bit_endian not in ('big', 'little'): raise TypeError( 'bit_endian should be either \'big\' or \'little\'') self.bit_endian = bit_endian if io_endian not in ('big', 'little'): raise TypeError('io_endian should be either \'big\' or \'little\'') self.io_endian = io_endian if start_offset is None: self.byte_pos = len(buffer) - 1 if bytes_reverse else 0 self.bit_pos = 0 if bit_endian == 'big' else 7 elif isinstance(start_offset, int): self.byte_pos = start_offset self.bit_pos = 0 if bit_endian == 'big' else 7 elif isinstance(start_offset, tuple): self.byte_pos, self.bit_pos = start_offset else: raise TypeError('start_offset should be of type int or tuple')
def histdump_iter( source, start=None, end=None, length=None, samples=0x10000, width=64, address_base=None ): assert is_bytes( source ) start, end = bounds( start, end, length, len( source ) ) start = max( start, 0 ) end = min( end, len( source ) ) if len( source ) == 0 or (start == end == 0): return address_base_offset = address_base-start if address_base is not None else 0 for offset in range( start, end, samples ): yield ansi.format_histdump_line( source, offset, length=samples, end=end, width=width, address_base_offset=address_base_offset ) return
def normalise_audio( source, format_type, field_size, signedness, endian, start=None, end=None, length=None ): assert is_bytes( source ) start, end = bounds( start, end, length, len( source ) ) if format_type == float: return array( 'f', encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] ) ) elif format_type == int: divisor = 1 << (field_size*8-1) if signedness == 'signed': return array( 'f', (float( x )/divisor for x in encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] )) ) else: return array( 'f', (float( x-divisor )/divisor for x in encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] )) ) return array( 'f' )
def import_data( self, raw_buffer ): """Import data from a byte array. raw_buffer Byte array to import from. """ klass = self.__class__ if raw_buffer: assert common.is_bytes( raw_buffer ) # raw_buffer = memoryview( raw_buffer ) self._field_data = {} for name in klass._fields: if raw_buffer: self._field_data[name] = klass._fields[name].get_from_buffer( raw_buffer, parent=self ) else: self._field_data[name] = klass._fields[name].default if raw_buffer: for name, check in klass._checks.items(): check.check_buffer( raw_buffer, parent=self ) # if we have debug logging on, check the roundtrip works if logger.isEnabledFor( logging.INFO ): test = self.export_data() if logger.getEffectiveLevel() <= logging.DEBUG: logger.debug( 'Stats for {}:'.format( self ) ) logger.debug( 'Import buffer size: {}'.format( len( raw_buffer ) ) ) logger.debug( 'Export size: {}'.format( len( test ) ) ) if test == raw_buffer: logger.debug( 'Content: exact match!' ) elif test == raw_buffer[:len( test )]: logger.debug( 'Content: partial match!' ) else: logger.debug( 'Content: different!' ) for x in utils.hexdump_diff_iter( raw_buffer[:len( test )], test ): logger.debug( x ) elif test != raw_buffer[:len( test )]: logger.info( '{} export produced changed output from import'.format( self ) ) # if raw_buffer: # raw_buffer.release() return
def hexdump_iter( source, start=None, end=None, length=None, major_len=8, minor_len=4, colour=True, address_base=None ): """Return the contents of a byte string in tabular hexadecimal/ASCII format. source The byte string to print. start Start offset to read from (default: start) end End offset to stop reading at (default: end) length Length to read in (optional replacement for end) major_len Number of hexadecimal groups per line minor_len Number of bytes per hexadecimal group colour Add ANSI colour formatting to output (default: true) address_base Base address to use for labels (default: start) Raises ValueError if both end and length are defined. """ assert is_bytes( source ) start, end = bounds( start, end, length, len( source ) ) start = max( start, 0 ) end = min( end, len( source ) ) if len( source ) == 0 or (start == end == 0): return address_base_offset = address_base-start if address_base is not None else 0 for offset in range( start, end, minor_len*major_len ): yield ansi.format_hexdump_line( source, offset, end, major_len, minor_len, colour, address_base_offset=address_base_offset ) return
def __init__(self, buffer, start_offset, bytes_reverse=False, bits_reverse=False, output_reverse=False, bytes_to_cache=1): """Create a BitReader instance. buffer Source byte string to read from. start_offset Position in the block to start reading from. bytes_reverse If enabled, fetch successive bytes from the source in reverse order. bits_reverse If enabled, fetch bits starting from the most-significant bit (0x80) through least-significant bit (0x01). output_reverse If enabled, return fetched bits starting from the most-significant bit (0x80) through least-significant bit (0x01). bytes_to_cache Number of bytes to cache. Defaults to 1. Only useful for algorithms which change the position pointer mid-read. """ assert is_bytes(buffer) assert start_offset in range(len(buffer)) self.buffer = buffer self.bits_reverse = bits_reverse self.bytes_reverse = bytes_reverse self.output_reverse = output_reverse self.pos = start_offset self.bytes_to_cache = bytes_to_cache self._fill_buffer()
def play_pcm( source, channels, sample_rate, format_type, field_size, signedness, endian, start=None, end=None, length=None, interpolation=AudioInterpolation.LINEAR ): assert is_bytes( source ) start, end = bounds( start, end, length, len( source ) ) if not pyaudio: raise ImportError( 'pyaudio must be installed for audio playback support (see https://people.csail.mit.edu/hubert/pyaudio)' ) audio = pyaudio.PyAudio() format = getattr( pyaudio, PYAUDIO_NORMALISE_TYPE ) rate = sample_rate samp_iter = resample_audio_iter( source, format_type, field_size, signedness, endian, channels, sample_rate, start, end, output_rate=RESAMPLE_RATE ) stream = audio.open( format=format, channels=channels, rate=RESAMPLE_RATE, output=True ) for samples in samp_iter: stream.write( samples.tobytes() ) stream.stop_stream() stream.close()
def play_pcm(source, channels, sample_rate, format_type, field_size, signedness, endian, start=None, end=None, length=None, interpolation=AudioInterpolation.LINEAR): """Play back a byte string as PCM audio. source The byte string to play. channels Number of audio channels. sample_rate Audio sample rate in Hz. format_type Type of sample encoding; either int or float. field_size Size of each sample, in bytes. signedness Signedness of each sample; either 'signed' or 'unsigned'. endian Endianness of each sample; either 'big', 'little' or None. start Start offset to read from (default: start). end End offset to stop reading at (default: end). length Length to read in (optional replacement for end). interpolation Interpolation algorithm to use for upsampling. Defaults to AudioInterpolation.LINEAR. """ assert is_bytes(source) start, end = bounds(start, end, length, len(source)) if not miniaudio: raise ImportError( 'miniaudio must be installed for audio playback support (see https://github.com/irmen/pyminiaudio)' ) format = getattr(miniaudio.SampleFormat, MINIAUDIO_NORMALISE_TYPE) playback_rate = None INTERP_MAP = { AudioInterpolation.NONE: miniaudio.DitherMode.NONE, AudioInterpolation.LINEAR: miniaudio.DitherMode.TRIANGLE, AudioInterpolation.STEP: miniaudio.DitherMode.RECTANGLE, } interpolation = INTERP_MAP.get(interpolation, miniaudio.DitherMode.NONE) FORMAT_MAP = { (int, 1, 'unsigned', None): miniaudio.SampleFormat.UNSIGNED8, (int, 2, 'signed', 'little'): miniaudio.SampleFormat.SIGNED16, (int, 3, 'signed', 'little'): miniaudio.SampleFormat.SIGNED24, (int, 4, 'signed', 'little'): miniaudio.SampleFormat.SIGNED32, (float, 4, 'signed', 'little'): miniaudio.SampleFormat.FLOAT32, } format = FORMAT_MAP.get((format_type, field_size, signedness, endian)) if not format: raise ValueError('Format not supported yet!') with miniaudio.PlaybackDevice(output_format=format, nchannels=channels, sample_rate=PLAYBACK_RATE) as device: def audio_iter(): conv = miniaudio.convert_frames(format, channels, sample_rate, source[start:end], device.format, device.nchannels, device.sample_rate) samp_iter = iter(conv) required_frames = yield b'' old_time = time.time() while True: sample_data = bytes( itertools.islice(samp_iter, required_frames * channels * field_size)) if not sample_data: break new_time = time.time() old_time = new_time required_frames = yield sample_data ai = audio_iter() next(ai) device.start(ai) while device.callback_generator: time.sleep(0.1)
def play_pcm(source, channels, sample_rate, format_type, field_size, signedness, endian, start=None, end=None, length=None, interpolation=AudioInterpolation.LINEAR): """Play back a byte string as PCM audio. source The byte string to play. channels Number of audio channels. sample_rate Audio sample rate in Hz. format_type Type of sample encoding; either int or float. field_size Size of each sample, in bytes. signedness Signedness of each sample; either 'signed' or 'unsigned'. endian Endianness of each sample; either 'big', 'little' or None. start Start offset to read from (default: start). end End offset to stop reading at (default: end). length Length to read in (optional replacement for end). interpolation Interpolation algorithm to use for upsampling. Defaults to AudioInterpolation.LINEAR. """ assert is_bytes(source) start, end = bounds(start, end, length, len(source)) if not pyaudio: raise ImportError( 'pyaudio must be installed for audio playback support (see https://people.csail.mit.edu/hubert/pyaudio)' ) audio = pyaudio.PyAudio() format = getattr(pyaudio, PYAUDIO_NORMALISE_TYPE) playback_rate = None if interpolation == AudioInterpolation.NONE: playback_rate = sample_rate else: playback_rate = RESAMPLE_RATE samp_iter = resample_audio_iter(source, format_type, field_size, signedness, endian, channels, sample_rate, start, end, output_rate=playback_rate, interpolation=interpolation) stream = audio.open(format=format, channels=channels, rate=playback_rate, output=True) for samples in samp_iter: stream.write(samples.tobytes()) stream.stop_stream() stream.close()