def __init__( self, buffer, start_offset, bytes_reverse=False, bits_reverse=False, output_reverse=False, bytes_to_cache=1 ):
"""Create a BitReader instance.
buffer
Source byte string to read from.
start_offset
Position in the block to start reading from.
bytes_reverse
If enabled, fetch successive bytes from the source in reverse order.
bits_reverse
If enabled, fetch bits starting from the most-significant bit (i.e. 0x80)
through least-significant bit (0x01).
output_reverse
If enabled, return fetched bits starting from the most-significant bit (e.g.
0x80) through least-significant bit (0x01).
bytes_to_cache
Number of bytes to cache. Defaults to 1. Only useful for algorithms which
change the position pointer mid-read.
"""
assert is_bytes( buffer )
assert start_offset in range( len( buffer ) )
self.buffer = buffer
self.bits_reverse = bits_reverse
self.bytes_reverse = bytes_reverse
self.output_reverse = output_reverse
self.pos = start_offset
self.bytes_to_cache = bytes_to_cache
self._fill_buffer()
def __init__(self,
source_data=None,
parent=None,
preload_attrs=None,
endian=None,
cache_bytes=False):
"""Base class for Blocks.
source_data
Source data to construct Block with. Can be a byte string, dictionary
of attribute: value pairs, or another Block object.
parent
Parent Block object where this Block is defined. Used for e.g.
evaluating Refs.
preload_attrs
Attributes on the Block to set before importing the data. Used
for linking in dependencies before loading.
endian
Platform endianness to use when interpreting the Block data.
Useful for Blocks which have the same data layout but different
endianness for stored numbers. Has no effect on fields with an
predefined endianness.
cache_bytes
Cache the bytes equivalent of the Block. Useful for debugging the
loading procedure. Defaults to False.
"""
self._field_data = {}
self._ref_cache = {}
if parent is not None:
assert isinstance(parent, Block)
self._parent = parent
self._endian = endian if endian else (
parent._endian if parent else self._endian)
if cache_bytes:
self._cache_bytes = True
if source_data and common.is_bytes(source_data):
self._bytes = bytes(source_data)
if preload_attrs:
for attr, value in preload_attrs.items():
setattr(self, attr, value)
# start the initial load of data
if isinstance(source_data, Block):
self.clone_data(source_data)
elif isinstance(source_data, dict):
# preload defaults, then overwrite with dictionary values
self.import_data(None)
self.update_data(source_data)
else:
self.import_data(source_data)
# cache all refs
for key, ref in self._refs.items():
ref.cache(self)
def histdump_iter(source,
start=None,
end=None,
length=None,
samples=0x10000,
width=64,
address_base=None):
assert is_bytes(source)
start, end = bounds(start, end, length, len(source))
start = max(start, 0)
end = min(end, len(source))
if len(source) == 0 or (start == end == 0):
return
address_base_offset = address_base - start if address_base is not None else 0
for offset in range(start, end, samples):
yield ansi.format_histdump_line(
source,
offset,
length=samples,
end=end,
width=width,
address_base_offset=address_base_offset)
return
def pixdump_iter(source,
start=None,
end=None,
length=None,
width=64,
height=None,
palette=None):
"""Return the contents of a byte string as a 256 colour image.
source
The byte string to print.
start
Start offset to read from (default: start)
end
End offset to stop reading at (default: end)
length
Length to read in (optional replacement for end)
width
Width of image to render in pixels (default: 64)
height
Height of image to render in pixels (default: auto)
palette
List of Colours to use (default: test palette)
"""
assert is_bytes(source)
if not palette:
palette = colour.TEST_PALETTE
start = 0 if (start is None) else start
if (end is not None) and (length is not None):
raise ValueError('Can\'t define both an end and a length!')
elif (length is not None):
end = start + length
elif (end is not None):
pass
else:
end = len(source)
start = max(start, 0)
end = min(end, len(source))
if len(source) == 0 or (start == end == 0):
return iter(())
if height is None:
height = math.ceil((end - start) / width)
def data_fetch(x_pos, y_pos, frame):
index = y_pos * width + x_pos + start
if index >= end:
return (0, 0, 0, 0)
return palette[source[index]]
return ansi.format_image_iter(data_fetch, width=width, height=height)
def __init__(self,
source,
start=None,
end=None,
length=None,
major_len=8,
minor_len=4,
colour=True,
address_base=None,
before=2,
after=2,
title=None):
assert is_bytes(source)
self.source = source
self.start, self.end = bounds(start, end, length, len(source))
if len(source) == 0 or (start == end == 0):
return
self.address_base_offset = address_base - start if address_base is not None else 0
self.major_len = major_len
self.minor_len = minor_len
self.colour = colour
self.before = before
self.after = after
self.title = title
self.stride = minor_len * major_len
self.lines = []
self.last_printed = -1
self.output_buffer = {}
self.printed = False
def normalise_audio_iter( source, format_type, field_size, signedness, endian, start=None, end=None, length=None, overlap=0, chunk_size=NORMALIZE_BUFFER ):
assert is_bytes( source )
start, end = bounds( start, end, length, len( source ) )
increment = chunk_size+overlap
for i in range( start, end, chunk_size ):
yield normalise_audio( source, format_type, field_size, signedness, endian, start=i, end=None, length=increment )
def import_data(self, raw_buffer):
"""Import data from a byte array.
raw_buffer
Byte array to import from.
"""
klass = self.__class__
if raw_buffer is not None:
assert common.is_bytes(raw_buffer)
# raw_buffer = memoryview( raw_buffer )
self._field_data = {}
logger.debug('{}: loading fields'.format(self))
for name in klass._fields:
if raw_buffer is not None:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('{} [{}]: input buffer'.format(
name, klass._fields[name]))
self._field_data[name] = klass._fields[name].get_from_buffer(
raw_buffer, parent=self)
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Result for {} [{}]: {}'.format(
name, klass._fields[name], self._field_data[name]))
else:
self._field_data[name] = klass._fields[name].default
if raw_buffer is not None:
for name, check in klass._checks.items():
check.check_buffer(raw_buffer, parent=self)
# if we have debug logging on, check the roundtrip works
if logger.isEnabledFor(logging.INFO):
test = self.export_data()
if logger.getEffectiveLevel() <= logging.DEBUG:
logger.debug('Stats for {}:'.format(self))
logger.debug('Import buffer size: {}'.format(
len(raw_buffer)))
logger.debug('Export size: {}'.format(len(test)))
if test == raw_buffer:
logger.debug('Content: exact match!')
elif test == raw_buffer[:len(test)]:
logger.debug('Content: exact match with overflow!')
else:
logger.debug('Content: different!')
for x in utils.hexdump_diff_iter(
raw_buffer[:len(test)], test):
logger.debug(x)
elif test != raw_buffer[:len(test)]:
logger.info(
'{} export produced changed output from import'.format(
self))
# if raw_buffer:
# raw_buffer.release()
return
def resample_audio_iter(source,
format_type,
field_size,
signedness,
endian,
channels,
sample_rate,
start=None,
end=None,
length=None,
interpolation=AudioInterpolation.LINEAR,
output_rate=RESAMPLE_RATE):
if sample_rate == 0:
yield 0.0
return
assert is_bytes(source)
start, end = bounds(start, end, length, len(source))
mixer = mix_linear
if interpolation == AudioInterpolation.STEP:
mixer = mix_step
new_len = (end - start) * output_rate // sample_rate
src_inc = NORMALIZE_BUFFER
chunk_size = src_inc * channels
src_iter = normalise_audio_iter(source,
format_type,
field_size,
signedness,
endian,
start,
end,
overlap=0,
chunk_size=chunk_size)
src = next(src_iter, None)
src_bound = src_inc
for index_base in range(0, new_len):
tgt_pos = index_base
src_pos = sample_rate * tgt_pos / output_rate
samp_index = math.floor(src_pos) % src_inc
alpha = math.fmod(src_pos, 1.0)
if src_pos > src_bound:
src = next(src_iter, None)
src_bound += src_inc
if src is None:
break
a = 0.0 if samp_index >= len(src) else src[samp_index]
b = 0.0 if samp_index + channels >= len(src) else src[samp_index +
channels]
yield mixer(a, b, alpha)
def pixdump_iter( source, start=None, end=None, length=None, width=64, height=None, palette=None ):
"""Return the contents of a byte string as a 256 colour image.
source
The byte string to print.
start
Start offset to read from (default: start)
end
End offset to stop reading at (default: end)
length
Length to read in (optional replacement for end)
width
Width of image to render in pixels (default: 64)
height
Height of image to render in pixels (default: auto)
palette
List of Colours to use (default: test palette)
"""
assert is_bytes( source )
if not palette:
palette = colour.TEST_PALETTE
start = 0 if (start is None) else start
if (end is not None) and (length is not None):
raise ValueError( 'Can\'t define both an end and a length!' )
elif (length is not None):
end = start+length
elif (end is not None):
pass
else:
end = len( source )
start = max( start, 0 )
end = min( end, len( source ) )
if len( source ) == 0 or (start == end == 0):
return iter(())
if height is None:
height = math.ceil( (end-start)/width )
def data_fetch( x_pos, y_pos, frame ):
index = y_pos*width + x_pos + start
if index >= end:
return (0, 0, 0, 0)
return palette[source[index]]
return ansi.format_image_iter( data_fetch, width=width, height=height )
def hexdump_iter(source,
start=None,
end=None,
length=None,
major_len=8,
minor_len=4,
colour=True,
address_base=None):
"""Return the contents of a byte string in tabular hexadecimal/ASCII format.
source
The byte string to print.
start
Start offset to read from (default: start)
end
End offset to stop reading at (default: end)
length
Length to read in (optional replacement for end)
major_len
Number of hexadecimal groups per line
minor_len
Number of bytes per hexadecimal group
colour
Add ANSI colour formatting to output (default: true)
address_base
Base address to use for labels (default: start)
Raises ValueError if both end and length are defined.
"""
assert is_bytes(source)
start, end = bounds(start, end, length, len(source))
start = max(start, 0)
end = min(end, len(source))
if len(source) == 0 or (start == end == 0):
return
address_base_offset = address_base - start if address_base is not None else 0
for offset in range(start, end, minor_len * major_len):
yield ansi.format_hexdump_line(source,
offset,
end,
major_len,
minor_len,
colour,
address_base_offset=address_base_offset)
return
def __init__(self,
buffer=None,
start_offset=None,
bytes_reverse=False,
bit_endian='big',
io_endian='big'):
"""Create a BitStream instance.
buffer
Target byte array to read/write from. Defaults to an empty array.
start_offset
Position in the target to start reading from. Can be an integer byte offset,
or a tuple containing the byte and bit offsets. Defaults to the start of the
stream, depending on the endianness and ordering options.
bytes_reverse
If enabled, fetch successive bytes from the source in reverse order.
bit_endian
Endianness of the backing storage; either 'big' or 'little'. Defaults to big
(i.e. starting from the most-significant bit (0x80) through least-significant
bit (0x10)).
io_endian
Endianness of data returned from read/write; either 'big' or 'little'. Defaults
to big (i.e. starting from the most-significant bit (0x80) through
least-significant bit (0x10)).
"""
if buffer is None:
self.buffer = bytearray()
else:
assert is_bytes(buffer)
self.buffer = buffer
self.bytes_reverse = bytes_reverse
if bit_endian not in ('big', 'little'):
raise TypeError(
'bit_endian should be either \'big\' or \'little\'')
self.bit_endian = bit_endian
if io_endian not in ('big', 'little'):
raise TypeError('io_endian should be either \'big\' or \'little\'')
self.io_endian = io_endian
if start_offset is None:
self.byte_pos = len(buffer) - 1 if bytes_reverse else 0
self.bit_pos = 0 if bit_endian == 'big' else 7
elif isinstance(start_offset, int):
self.byte_pos = start_offset
self.bit_pos = 0 if bit_endian == 'big' else 7
elif isinstance(start_offset, tuple):
self.byte_pos, self.bit_pos = start_offset
else:
raise TypeError('start_offset should be of type int or tuple')
def histdump_iter( source, start=None, end=None, length=None, samples=0x10000, width=64, address_base=None ):
assert is_bytes( source )
start, end = bounds( start, end, length, len( source ) )
start = max( start, 0 )
end = min( end, len( source ) )
if len( source ) == 0 or (start == end == 0):
return
address_base_offset = address_base-start if address_base is not None else 0
for offset in range( start, end, samples ):
yield ansi.format_histdump_line( source, offset, length=samples, end=end, width=width, address_base_offset=address_base_offset )
return
def normalise_audio( source, format_type, field_size, signedness, endian, start=None, end=None, length=None ):
assert is_bytes( source )
start, end = bounds( start, end, length, len( source ) )
if format_type == float:
return array( 'f', encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] ) )
elif format_type == int:
divisor = 1 << (field_size*8-1)
if signedness == 'signed':
return array( 'f', (float( x )/divisor for x in encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] )) )
else:
return array( 'f', (float( x-divisor )/divisor for x in encoding.unpack_array( (format_type, field_size, signedness, endian), source[start:end] )) )
return array( 'f' )
def import_data( self, raw_buffer ):
"""Import data from a byte array.
raw_buffer
Byte array to import from.
"""
klass = self.__class__
if raw_buffer:
assert common.is_bytes( raw_buffer )
# raw_buffer = memoryview( raw_buffer )
self._field_data = {}
for name in klass._fields:
if raw_buffer:
self._field_data[name] = klass._fields[name].get_from_buffer(
raw_buffer, parent=self
)
else:
self._field_data[name] = klass._fields[name].default
if raw_buffer:
for name, check in klass._checks.items():
check.check_buffer( raw_buffer, parent=self )
# if we have debug logging on, check the roundtrip works
if logger.isEnabledFor( logging.INFO ):
test = self.export_data()
if logger.getEffectiveLevel() <= logging.DEBUG:
logger.debug( 'Stats for {}:'.format( self ) )
logger.debug( 'Import buffer size: {}'.format( len( raw_buffer ) ) )
logger.debug( 'Export size: {}'.format( len( test ) ) )
if test == raw_buffer:
logger.debug( 'Content: exact match!' )
elif test == raw_buffer[:len( test )]:
logger.debug( 'Content: partial match!' )
else:
logger.debug( 'Content: different!' )
for x in utils.hexdump_diff_iter( raw_buffer[:len( test )], test ):
logger.debug( x )
elif test != raw_buffer[:len( test )]:
logger.info( '{} export produced changed output from import'.format( self ) )
# if raw_buffer:
# raw_buffer.release()
return
def hexdump_iter( source, start=None, end=None, length=None, major_len=8, minor_len=4, colour=True, address_base=None ):
"""Return the contents of a byte string in tabular hexadecimal/ASCII format.
source
The byte string to print.
start
Start offset to read from (default: start)
end
End offset to stop reading at (default: end)
length
Length to read in (optional replacement for end)
major_len
Number of hexadecimal groups per line
minor_len
Number of bytes per hexadecimal group
colour
Add ANSI colour formatting to output (default: true)
address_base
Base address to use for labels (default: start)
Raises ValueError if both end and length are defined.
"""
assert is_bytes( source )
start, end = bounds( start, end, length, len( source ) )
start = max( start, 0 )
end = min( end, len( source ) )
if len( source ) == 0 or (start == end == 0):
return
address_base_offset = address_base-start if address_base is not None else 0
for offset in range( start, end, minor_len*major_len ):
yield ansi.format_hexdump_line( source, offset, end, major_len, minor_len, colour, address_base_offset=address_base_offset )
return
def __init__(self,
buffer,
start_offset,
bytes_reverse=False,
bits_reverse=False,
output_reverse=False,
bytes_to_cache=1):
"""Create a BitReader instance.
buffer
Source byte string to read from.
start_offset
Position in the block to start reading from.
bytes_reverse
If enabled, fetch successive bytes from the source in reverse order.
bits_reverse
If enabled, fetch bits starting from the most-significant bit (0x80)
through least-significant bit (0x01).
output_reverse
If enabled, return fetched bits starting from the most-significant bit
(0x80) through least-significant bit (0x01).
bytes_to_cache
Number of bytes to cache. Defaults to 1. Only useful for algorithms which
change the position pointer mid-read.
"""
assert is_bytes(buffer)
assert start_offset in range(len(buffer))
self.buffer = buffer
self.bits_reverse = bits_reverse
self.bytes_reverse = bytes_reverse
self.output_reverse = output_reverse
self.pos = start_offset
self.bytes_to_cache = bytes_to_cache
self._fill_buffer()
def play_pcm( source, channels, sample_rate, format_type, field_size, signedness, endian, start=None, end=None, length=None, interpolation=AudioInterpolation.LINEAR ):
assert is_bytes( source )
start, end = bounds( start, end, length, len( source ) )
if not pyaudio:
raise ImportError( 'pyaudio must be installed for audio playback support (see https://people.csail.mit.edu/hubert/pyaudio)' )
audio = pyaudio.PyAudio()
format = getattr( pyaudio, PYAUDIO_NORMALISE_TYPE )
rate = sample_rate
samp_iter = resample_audio_iter( source, format_type, field_size, signedness, endian, channels, sample_rate, start, end, output_rate=RESAMPLE_RATE )
stream = audio.open(
format=format,
channels=channels,
rate=RESAMPLE_RATE,
output=True
)
for samples in samp_iter:
stream.write( samples.tobytes() )
stream.stop_stream()
stream.close()
def play_pcm(source,
channels,
sample_rate,
format_type,
field_size,
signedness,
endian,
start=None,
end=None,
length=None,
interpolation=AudioInterpolation.LINEAR):
"""Play back a byte string as PCM audio.
source
The byte string to play.
channels
Number of audio channels.
sample_rate
Audio sample rate in Hz.
format_type
Type of sample encoding; either int or float.
field_size
Size of each sample, in bytes.
signedness
Signedness of each sample; either 'signed' or 'unsigned'.
endian
Endianness of each sample; either 'big', 'little' or None.
start
Start offset to read from (default: start).
end
End offset to stop reading at (default: end).
length
Length to read in (optional replacement for end).
interpolation
Interpolation algorithm to use for upsampling. Defaults to AudioInterpolation.LINEAR.
"""
assert is_bytes(source)
start, end = bounds(start, end, length, len(source))
if not miniaudio:
raise ImportError(
'miniaudio must be installed for audio playback support (see https://github.com/irmen/pyminiaudio)'
)
format = getattr(miniaudio.SampleFormat, MINIAUDIO_NORMALISE_TYPE)
playback_rate = None
INTERP_MAP = {
AudioInterpolation.NONE: miniaudio.DitherMode.NONE,
AudioInterpolation.LINEAR: miniaudio.DitherMode.TRIANGLE,
AudioInterpolation.STEP: miniaudio.DitherMode.RECTANGLE,
}
interpolation = INTERP_MAP.get(interpolation, miniaudio.DitherMode.NONE)
FORMAT_MAP = {
(int, 1, 'unsigned', None): miniaudio.SampleFormat.UNSIGNED8,
(int, 2, 'signed', 'little'): miniaudio.SampleFormat.SIGNED16,
(int, 3, 'signed', 'little'): miniaudio.SampleFormat.SIGNED24,
(int, 4, 'signed', 'little'): miniaudio.SampleFormat.SIGNED32,
(float, 4, 'signed', 'little'): miniaudio.SampleFormat.FLOAT32,
}
format = FORMAT_MAP.get((format_type, field_size, signedness, endian))
if not format:
raise ValueError('Format not supported yet!')
with miniaudio.PlaybackDevice(output_format=format,
nchannels=channels,
sample_rate=PLAYBACK_RATE) as device:
def audio_iter():
conv = miniaudio.convert_frames(format, channels, sample_rate,
source[start:end], device.format,
device.nchannels,
device.sample_rate)
samp_iter = iter(conv)
required_frames = yield b''
old_time = time.time()
while True:
sample_data = bytes(
itertools.islice(samp_iter,
required_frames * channels * field_size))
if not sample_data:
break
new_time = time.time()
old_time = new_time
required_frames = yield sample_data
ai = audio_iter()
next(ai)
device.start(ai)
while device.callback_generator:
time.sleep(0.1)
def play_pcm(source,
channels,
sample_rate,
format_type,
field_size,
signedness,
endian,
start=None,
end=None,
length=None,
interpolation=AudioInterpolation.LINEAR):
"""Play back a byte string as PCM audio.
source
The byte string to play.
channels
Number of audio channels.
sample_rate
Audio sample rate in Hz.
format_type
Type of sample encoding; either int or float.
field_size
Size of each sample, in bytes.
signedness
Signedness of each sample; either 'signed' or 'unsigned'.
endian
Endianness of each sample; either 'big', 'little' or None.
start
Start offset to read from (default: start).
end
End offset to stop reading at (default: end).
length
Length to read in (optional replacement for end).
interpolation
Interpolation algorithm to use for upsampling. Defaults to AudioInterpolation.LINEAR.
"""
assert is_bytes(source)
start, end = bounds(start, end, length, len(source))
if not pyaudio:
raise ImportError(
'pyaudio must be installed for audio playback support (see https://people.csail.mit.edu/hubert/pyaudio)'
)
audio = pyaudio.PyAudio()
format = getattr(pyaudio, PYAUDIO_NORMALISE_TYPE)
playback_rate = None
if interpolation == AudioInterpolation.NONE:
playback_rate = sample_rate
else:
playback_rate = RESAMPLE_RATE
samp_iter = resample_audio_iter(source,
format_type,
field_size,
signedness,
endian,
channels,
sample_rate,
start,
end,
output_rate=playback_rate,
interpolation=interpolation)
stream = audio.open(format=format,
channels=channels,
rate=playback_rate,
output=True)
for samples in samp_iter:
stream.write(samples.tobytes())
stream.stop_stream()
stream.close()