def __init__(self):
super(VideoHandler, self).__init__()
self.last_sent_time = 0
self.decoder = H264Decoder()
self.header = construct.BitStruct(
'VSTRMHeader', construct.Nibble('magic'),
construct.BitField('packet_type', 2),
construct.BitField('seq_id', 10), construct.Flag('init'),
construct.Flag('frame_begin'), construct.Flag('chunk_end'),
construct.Flag('frame_end'), construct.Flag('has_timestamp'),
construct.BitField('payload_size', 11),
construct.BitField('timestamp', 32))
self.frame = array.array('B')
self.is_streaming = False
self.frame_decode_num = 0
def __init__(self):
super(AudioHandler, self).__init__()
self.header_base = construct.BitStruct(
'ASTRMBaseHeader', construct.BitField('fmt', 3),
construct.Bit('channel'), construct.Flag('vibrate'),
construct.Bit('packet_type'), construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16))
self.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'))
self.header_msg = construct.Struct(
'ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used
# depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3))
self.header = construct.Struct(
'ASTRMHeader', construct.Embed(self.header_base),
construct.Switch('format_hdr',
lambda ctx: ctx.packet_type, {
0: construct.Embed(self.header_aud),
1: construct.Embed(self.header_msg),
},
default=construct.Pass))
def __init__(s):
super(ServiceVSTRM, s).__init__()
s.decoder = H264Decoder(s.dimensions['gamepad'],
pygame.display.get_surface().get_size())
s.header = construct.BitStruct('VSTRMHeader',
construct.Nibble('magic'),
construct.BitField('packet_type', 2),
construct.BitField('seq_id', 10),
construct.Flag('init'),
construct.Flag('frame_begin'),
construct.Flag('chunk_end'),
construct.Flag('frame_end'),
construct.Flag('has_timestamp'),
construct.BitField('payload_size', 11),
construct.BitField('timestamp', 32))
s.frame = array.array('B')
s.is_streaming = False
s.frame_decode_num = 0
def __init__(s, decoder):
super(ServiceVSTRM, s).__init__()
s.decoder = decoder
s.header = construct.BitStruct('VSTRMHeader',
construct.Nibble('magic'),
construct.BitField('packet_type', 2),
construct.BitField('seq_id', 10),
construct.Flag('init'),
construct.Flag('frame_begin'),
construct.Flag('chunk_end'),
construct.Flag('frame_end'),
construct.Flag('has_timestamp'),
construct.BitField('payload_size', 11),
construct.BitField('timestamp', 32))
s.frame = array.array('B')
s._is_streaming = False
s.is_streaming = False
s.frame_decode_num = 0
def __init__(s):
super(ServiceASTRM, s).__init__()
s.header_base = construct.BitStruct('ASTRMBaseHeader',
construct.BitField('fmt', 3),
construct.Bit('channel'),
construct.Flag('vibrate'),
construct.Bit('packet_type'),
construct.BitField('seq_id', 10),
construct.BitField('payload_size', 16)
)
s.header_aud = construct.Struct('ASTRMAudioHeader',
construct.ULInt32('timestamp'),
# construct.Array(lambda ctx: ctx.payload_size, construct.UBInt8("data"))
)
s.header_msg = construct.Struct('ASTRMMsgHeader',
# This is kind of a hack, (there are two timestamp fields, which one is used depends on packet_type
construct.ULInt32('timestamp_audio'),
construct.ULInt32('timestamp'),
construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video
construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync
construct.ULInt8('vid_format'),
construct.Padding(3)
)
s.header = construct.Struct('ASTRMHeader',
construct.Embed(s.header_base),
construct.Switch('format_hdr', lambda ctx: ctx.packet_type,
{
0 : construct.Embed(s.header_aud),
1 : construct.Embed(s.header_msg),
},
default = construct.Pass
)
)
s.is_streaming = False
s.p = pyaudio.PyAudio()
s.stream = None
s.pa_num_bufs = 15
s.pa_ring = [array.array('H', '\0' * 416 * 2)] * s.pa_num_bufs
s.pa_wpos = s.pa_rpos = 0
"""
Setup and unique functionality for the wide-band correlator modes. A wideband correlator's FPGAs process all digitised data, which is a multiple of the FPGA clock rates.
"""
"""
Revisions:
2011-07-07 PVP Initial revision.
"""
import construct, corr_functions
# f-engine control register
register_fengine_control = construct.BitStruct(
'control',
construct.Padding(32 - 20 - 1), # 21 - 31
construct.Flag('tvgsel_noise'), # 20
construct.Flag('tvgsel_fdfs'), # 19
construct.Flag('tvgsel_pkt'), # 18
construct.Flag('tvgsel_ct'), # 17
construct.Flag('tvg_en'), # 16
construct.Padding(16 - 13 - 1), # 14 - 15
construct.Flag('adc_protect_disable'), # 13
construct.Flag('flasher_en'), # 12
construct.Padding(12 - 9 - 1), # 10 - 11
construct.Flag('gbe_enable'), # 9
construct.Flag('gbe_rst'), # 8
construct.Padding(8 - 3 - 1), # 4 - 7
construct.Flag('clr_status'), # 3
construct.Flag('arm'), # 2
construct.Flag('soft_sync'), # 1
construct.Flag('mrst')) # 0
# f-engine status
import construct as cst
import time
import sys
con = cst.Container # alias
HEADER = cst.Struct('pc_header', cst.Magic('\x00'),
cst.Const(cst.UBInt8('lines'), 1), cst.UBInt8('address'),
cst.Magic('\x03'))
SER_STATUS = cst.BitStruct('serst', cst.Magic('\x01\x01\x00'),
cst.Flag('schedule_enabled'),
cst.Flag('ack_enabled'), cst.Flag('further_pages'),
cst.Flag('interrupt_mode'), cst.Magic('\x00'))
PAGE_IDX = cst.Bytes('page_num', 3)
TEMPO = cst.BitStruct(
'tempo', cst.Magic('\x01\x01'),
cst.Enum(cst.BitField('display_ctrl', 2), TIMED=0, FIXED_ON=1,
FIXED_OFF=2),
cst.Enum(cst.BitField('persist_time', 4),
S2=1,
S5=2,
S10=3,
S20=4,
S30=5,
S45=6,
S60=7,
S90=8,
S120=9))
class AslParser(interface.BaseParser):
"""Parser for ASL log files."""
NAME = 'asl_log'
DESCRIPTION = u'Parser for ASL log files.'
ASL_MAGIC = 'ASL DB\x00\x00\x00\x00\x00\x00'
# If not right assigned, the value is "-1".
ASL_NO_RIGHTS = 'ffffffff'
# Priority level (criticity)
ASL_MESSAGE_PRIORITY = {
0: 'EMERGENCY',
1: 'ALERT',
2: 'CRITICAL',
3: 'ERROR',
4: 'WARNING',
5: 'NOTICE',
6: 'INFO',
7: 'DEBUG'
}
# ASL File header.
# magic: magic number that identify ASL files.
# version: version of the file.
# offset: first record in the file.
# timestamp: epoch time when the first entry was written.
# last_offset: last record in the file.
ASL_HEADER_STRUCT = construct.Struct('asl_header_struct',
construct.String('magic', 12),
construct.UBInt32('version'),
construct.UBInt64('offset'),
construct.UBInt64('timestamp'),
construct.UBInt32('cache_size'),
construct.UBInt64('last_offset'),
construct.Padding(36))
# The record structure is:
# [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry]
# Record static structure.
# tam_entry: it contains the number of bytes from this file position
# until the end of the record, without counts itself.
# next_offset: next record. If is equal to 0x00, it is the last record.
# asl_message_id: integer that has the numeric identification of the event.
# timestamp: Epoch integer that has the time when the entry was created.
# nanosecond: nanosecond to add to the timestamp.
# level: level of priority.
# pid: process identification that ask to save the record.
# uid: user identification that has lunched the process.
# gid: group identification that has lunched the process.
# read_uid: identification id of a user. Only applied if is not -1 (all FF).
# Only root and this user can read the entry.
# read_gid: the same than read_uid, but for the group.
ASL_RECORD_STRUCT = construct.Struct('asl_record_struct',
construct.Padding(2),
construct.UBInt32('tam_entry'),
construct.UBInt64('next_offset'),
construct.UBInt64('asl_message_id'),
construct.UBInt64('timestamp'),
construct.UBInt32('nanosec'),
construct.UBInt16('level'),
construct.UBInt16('flags'),
construct.UBInt32('pid'),
construct.UBInt32('uid'),
construct.UBInt32('gid'),
construct.UBInt32('read_uid'),
construct.UBInt32('read_gid'),
construct.UBInt64('ref_pid'))
ASL_RECORD_STRUCT_SIZE = ASL_RECORD_STRUCT.sizeof()
# 8-byte fields, they can be:
# - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String].
# - Integer: integer that has the byte position in the file that points
# to an ASL_RECORD_DYN_VALUE struct. If the value of the integer
# is equal to 0, it means that it has not data (skip).
# If the field is a String, we use this structure to decode each
# integer byte in the corresponding character (ASCII Char).
ASL_OCTET_STRING = construct.ExprAdapter(construct.Octet('string'),
encoder=lambda obj, ctx: ord(obj),
decoder=lambda obj, ctx: chr(obj))
# Field string structure. If the first bit is 1, it means that it
# is a String (1000) = 8, then the next nibble has the number of
# characters. The last 7 bytes are the number of bytes.
ASL_STRING = construct.BitStruct(
'string', construct.Flag('type'), construct.Bits('filler', 3),
construct.If(lambda ctx: ctx.type, construct.Nibble('string_length')),
construct.If(lambda ctx: ctx.type,
construct.Array(7, ASL_OCTET_STRING)))
# 8-byte pointer to a byte position in the file.
ASL_POINTER = construct.UBInt64('pointer')
# Dynamic data structure pointed by a pointer that contains a String:
# [2 bytes padding][4 bytes lenght of String][String].
ASL_RECORD_DYN_VALUE = construct.Struct(
'asl_record_dyn_value', construct.Padding(2),
construct.PascalString('value',
length_field=construct.UBInt32('length')))
def Parse(self, parser_context, file_entry):
"""Extract entries from an ASL file.
Args:
parser_context: A parser context object (instance of ParserContext).
file_entry: A file entry object (instance of dfvfs.FileEntry).
"""
file_object = file_entry.GetFileObject()
file_object.seek(0, os.SEEK_SET)
try:
header = self.ASL_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
file_object.close()
raise errors.UnableToParseFile(
u'Unable to parse ASL Header with error: {0:s}.'.format(
exception))
if header.magic != self.ASL_MAGIC:
file_object.close()
raise errors.UnableToParseFile(
u'Not an ASL Header, unable to parse.')
# Get the first and the last entry.
offset = header.offset
old_offset = header.offset
last_offset_header = header.last_offset
# If the ASL file has entries.
if offset:
event_object, offset = self.ReadAslEvent(file_object, offset)
while event_object:
parser_context.ProduceEvent(event_object,
parser_name=self.NAME,
file_entry=file_entry)
# TODO: an anomaly object must be emitted once that is implemented.
# Sanity check, the last read element must be the same as
# indicated by the header.
if offset == 0 and old_offset != last_offset_header:
logging.warning(u'Parsing ended before the header ends.')
old_offset = offset
event_object, offset = self.ReadAslEvent(file_object, offset)
file_object.close()
def ReadAslEvent(self, file_object, offset):
"""Returns an AslEvent from a single ASL entry.
Args:
file_object: a file-like object that points to an ASL file.
offset: offset where the static part of the entry starts.
Returns:
An event object constructed from a single ASL record.
"""
# The heap of the entry is saved to try to avoid seek (performance issue).
# It has the real start position of the entry.
dynamic_start = file_object.tell()
dynamic_part = file_object.read(offset - file_object.tell())
if not offset:
return None, None
try:
record_header = self.ASL_RECORD_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
# Variable tam_fields = is the real length of the dynamic fields.
# We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before]
# In Record_Struct we have a field called tam_entry, where it has the number
# of bytes until the end of the entry from the position that the field is.
# The tam_entry is between the 2th and the 6th byte in the [Record_Struct].
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# Also, we do not need [Point_Entry_Before] and then we delete the size of
# [Point_Entry_Before] that it is 8 bytes (8):
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8
# [Dynamic_Fields] = tam_entry - [Record_Struct] - 2
tam_fields = record_header.tam_entry - self.ASL_RECORD_STRUCT_SIZE - 2
# Dynamic part of the entry that contains minimal four fields of 8 bytes
# plus 2x[8bytes] fields for each extra ASL_Field.
# The four first fields are always the Host, Sender, Facility and Message.
# After the four first fields, the entry might have extra ASL_Fields.
# For each extra ASL_field, it has a pair of 8-byte fields where the first
# 8 bytes contains the name of the extra ASL_field and the second 8 bytes
# contains the text of the exta field.
# All of this 8-byte field can be saved using one of these three differents
# types:
# - Null value ('0000000000000000'): nothing to do.
# - String: It is string if first bit = 1 or first nibble = 8 (1000).
# Second nibble has the length of string.
# The next 7 bytes have the text characters of the string
# padding the end with null characters: '0x00'.
# Example: [8468 6964 6400 0000]
# [8] String, [4] length, value: [68 69 64 64] = hidd.
# - Pointer: static position in the file to a special struct
# implemented as an ASL_RECORD_DYN_VALUE.
# Example: [0000 0000 0000 0077]
# It points to the file position 0x077 that has a
# ASL_RECORD_DYN_VALUE structure.
values = []
while tam_fields > 0:
try:
raw_field = file_object.read(8)
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:d}'.format(
exception))
return None, None
try:
# Try to read as a String.
field = self.ASL_STRING.parse(raw_field)
values.append(''.join(field.string[0:field.string_length]))
# Go to parse the next extra field.
tam_fields -= 8
continue
except ValueError:
pass
# If it is not a string, it must be a pointer.
try:
field = self.ASL_POINTER.parse(raw_field)
except ValueError as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.format(
exception))
return None, None
if field != 0:
# The next IF ELSE is only for performance issues, avoiding seek.
# If the pointer points a lower position than where the actual entry
# starts, it means that it points to a previuos entry.
pos = field - dynamic_start
# Bigger or equal 0 means that the data is in the actual entry.
if pos >= 0:
try:
values.append((self.ASL_RECORD_DYN_VALUE.parse(
dynamic_part[pos:])).value.partition('\x00')[0])
except (IOError, construct.FieldError) as exception:
logging.warning(
u'Unable to parse ASL event with error: {0:s}'.
format(exception))
return None, None
else:
# Only if it is a pointer that points to the
# heap from another entry we use the seek method.
main_position = file_object.tell()
# If the pointer is in a previous entry.
if main_position > field:
file_object.seek(field - main_position, os.SEEK_CUR)
try:
values.append(
(self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
except (IOError, construct.FieldError):
logging.warning((
u'The pointer at {0:d} (0x{0:x}) points to invalid '
u'information.'
).format(main_position -
self.ASL_POINTER.sizeof()))
# Come back to the position in the entry.
_ = file_object.read(main_position -
file_object.tell())
else:
_ = file_object.read(field - main_position)
values.append((self.ASL_RECORD_DYN_VALUE.parse_stream(
file_object)).value.partition('\x00')[0])
# Come back to the position in the entry.
file_object.seek(main_position - file_object.tell(),
os.SEEK_CUR)
# Next extra field: 8 bytes more.
tam_fields -= 8
# Read the last 8 bytes of the record that points to the previous entry.
_ = file_object.read(8)
# Parsed section, we translate the read data to an appropriate format.
microsecond = record_header.nanosec // 1000
timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond(
record_header.timestamp, microsecond)
record_position = offset
message_id = record_header.asl_message_id
level = u'{0} ({1})'.format(
self.ASL_MESSAGE_PRIORITY[record_header.level],
record_header.level)
# If the value is -1 (FFFFFFFF), it can be read by everyone.
if record_header.read_uid != int(self.ASL_NO_RIGHTS, 16):
read_uid = record_header.read_uid
else:
read_uid = 'ALL'
if record_header.read_gid != int(self.ASL_NO_RIGHTS, 16):
read_gid = record_header.read_gid
else:
read_gid = 'ALL'
# Parsing the dynamic values (text or pointers to position with text).
# The first four are always the host, sender, facility, and message.
computer_name = values[0]
sender = values[1]
facility = values[2]
message = values[3]
# If the entry has an extra fields, they works as a pairs:
# The first is the name of the field and the second the value.
extra_information = ''
if len(values) > 4:
values = values[4:]
for index in xrange(0, len(values) // 2):
extra_information += (u'[{0}: {1}]'.format(
values[index * 2], values[(index * 2) + 1]))
# Return the event and the offset for the next entry.
return AslEvent(timestamp, record_position, message_id, level,
record_header, read_uid, read_gid, computer_name,
sender, facility, message,
extra_information), record_header.next_offset
decoder=lambda obj, ctx: ''.join(dsb.data_values
for dsb in obj.blocks),
)
_image_block = construct.Struct(
'image',
construct.Value('block_type', lambda ctx: 'image'),
construct.Struct(
'image_descriptor',
construct.ULInt16('left'),
construct.ULInt16('top'),
construct.ULInt16('width'),
construct.ULInt16('height'),
construct.EmbeddedBitStruct(
construct.Flag('lct_flag'),
construct.Flag('interlace_flag'),
construct.Flag('sort_flag'),
construct.Padding(2), # reserved
construct.macros.BitField('lct_size', 3),
),
),
construct.If(
lambda ctx: ctx.image_descriptor.lct_flag,
construct.Array(
lambda ctx: pow(2, ctx.image_descriptor.lct_size + 1),
construct.Array(3, construct.ULInt8('lct')),
),
),
construct.ULInt8('lzw_min'),
_get_data_subblocks('compressed_indices'),
class ASLParser(interface.FileObjectParser):
"""Parser for ASL log files."""
_INITIAL_FILE_OFFSET = None
NAME = u'asl_log'
DESCRIPTION = u'Parser for ASL log files.'
_ASL_MAGIC = b'ASL DB\x00\x00\x00\x00\x00\x00'
# ASL File header.
# magic: magic number that identify ASL files.
# version: version of the file.
# offset: first record in the file.
# timestamp: time when the first entry was written.
# Contains the number of seconds since January 1, 1970 00:00:00 UTC.
# last_offset: last record in the file.
_ASL_HEADER_STRUCT = construct.Struct(u'asl_header_struct',
construct.String(u'magic', 12),
construct.UBInt32(u'version'),
construct.UBInt64(u'offset'),
construct.UBInt64(u'timestamp'),
construct.UBInt32(u'cache_size'),
construct.UBInt64(u'last_offset'),
construct.Padding(36))
# The record structure is:
# [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry]
# Record static structure.
# tam_entry: it contains the number of bytes from this file position
# until the end of the record, without counts itself.
# next_offset: next record. If is equal to 0x00, it is the last record.
# asl_message_id: integer that has the numeric identification of the event.
# timestamp: the entry creation date and time.
# Contains the number of seconds since January 1, 1970 00:00:00 UTC.
# nanosecond: nanosecond to add to the timestamp.
# level: level of priority.
# pid: process identification that ask to save the record.
# uid: user identification that has lunched the process.
# gid: group identification that has lunched the process.
# read_uid: identification id of a user. Only applied if is not -1 (all FF).
# Only root and this user can read the entry.
# read_gid: the same than read_uid, but for the group.
_ASL_RECORD_STRUCT = construct.Struct(u'asl_record_struct',
construct.Padding(2),
construct.UBInt32(u'tam_entry'),
construct.UBInt64(u'next_offset'),
construct.UBInt64(u'asl_message_id'),
construct.UBInt64(u'timestamp'),
construct.UBInt32(u'nanosec'),
construct.UBInt16(u'level'),
construct.UBInt16(u'flags'),
construct.UBInt32(u'pid'),
construct.UBInt32(u'uid'),
construct.UBInt32(u'gid'),
construct.UBInt32(u'read_uid'),
construct.UBInt32(u'read_gid'),
construct.UBInt64(u'ref_pid'))
_ASL_RECORD_STRUCT_SIZE = _ASL_RECORD_STRUCT.sizeof()
# 8-byte fields, they can be:
# - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String].
# - Integer: integer that has the byte position in the file that points
# to an ASL_RECORD_DYN_VALUE struct. If the value of the integer
# is equal to 0, it means that it has not data (skip).
# If the field is a String, we use this structure to decode each
# integer byte in the corresponding character (ASCII Char).
_ASL_OCTET_STRING = construct.ExprAdapter(
construct.Octet(u'string'),
encoder=lambda obj, ctx: ord(obj),
decoder=lambda obj, ctx: chr(obj))
# Field string structure. If the first bit is 1, it means that it
# is a String (1000) = 8, then the next nibble has the number of
# characters. The last 7 bytes are the number of bytes.
_ASL_STRING = construct.BitStruct(
u'string', construct.Flag(u'type'), construct.Bits(u'filler', 3),
construct.If(lambda ctx: ctx.type, construct.Nibble(u'string_length')),
construct.If(lambda ctx: ctx.type,
construct.Array(7, _ASL_OCTET_STRING)))
# 8-byte pointer to a byte position in the file.
_ASL_POINTER = construct.UBInt64(u'pointer')
# Dynamic data structure pointed by a pointer that contains a String:
# [2 bytes padding][4 bytes size of String][String].
_ASL_RECORD_DYN_VALUE = construct.Struct(
u'asl_record_dyn_value', construct.Padding(2),
construct.UBInt32(u'size'),
construct.Bytes(u'value', lambda ctx: ctx.size))
def ParseFileObject(self, parser_mediator, file_object, **kwargs):
"""Parses an ALS file-like object.
Args:
parser_mediator: a parser mediator object (instance of ParserMediator).
file_object: a file-like object.
Raises:
UnableToParseFile: when the file cannot be parsed.
"""
file_object.seek(0, os.SEEK_SET)
try:
header = self._ASL_HEADER_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
raise errors.UnableToParseFile(
u'Unable to parse ASL Header with error: {0:s}.'.format(
exception))
if header.magic != self._ASL_MAGIC:
raise errors.UnableToParseFile(
u'Not an ASL Header, unable to parse.')
offset = header.offset
if not offset:
return
header_last_offset = header.last_offset
previous_offset = offset
event_object, offset = self.ReadASLEvent(parser_mediator, file_object,
offset)
while event_object:
# Sanity check, the last read element must be the same as
# indicated by the header.
if offset == 0 and previous_offset != header_last_offset:
parser_mediator.ProduceParseError(
u'Unable to parse header. Last element header does not match '
u'header offset.')
previous_offset = offset
event_object, offset = self.ReadASLEvent(parser_mediator,
file_object, offset)
def ReadASLEvent(self, parser_mediator, file_object, offset):
"""Reads an ASL record at a specific offset.
Args:
parser_mediator: a parser mediator object (instance of ParserMediator).
file_object: a file-like object that points to an ASL file.
offset: an integer containing the offset of the ASL record.
Returns:
A tuple of an event object extracted from the ASL record,
and the offset to the next ASL record in the file.
"""
# The heap of the entry is saved to try to avoid seek (performance issue).
# It has the real start position of the entry.
dynamic_data_offset = file_object.tell()
try:
dynamic_data = file_object.read(offset - dynamic_data_offset)
except IOError as exception:
parser_mediator.ProduceParseError(
u'unable to read ASL record dynamic data with error: {0:s}'.
format(exception))
return None, None
if not offset:
return None, None
try:
record_struct = self._ASL_RECORD_STRUCT.parse_stream(file_object)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceParseError(
u'unable to parse ASL record with error: {0:s}'.format(
exception))
return None, None
# Variable tam_fields = is the real length of the dynamic fields.
# We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before]
# In Record_Struct we have a field called tam_entry, where it has the number
# of bytes until the end of the entry from the position that the field is.
# The tam_entry is between the 2th and the 6th byte in the [Record_Struct].
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# Also, we do not need [Point_Entry_Before] and then we delete the size of
# [Point_Entry_Before] that it is 8 bytes (8):
# tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before]
# [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8
# [Dynamic_Fields] = tam_entry - [Record_Struct] - 2
tam_fields = record_struct.tam_entry - self._ASL_RECORD_STRUCT_SIZE - 2
# Dynamic part of the entry that contains minimal four fields of 8 bytes
# plus 2 x [8 bytes] fields for each extra ASL_Field.
# The four first fields are always the Host, Sender, Facility and Message.
# After the four first fields, the entry might have extra ASL_Fields.
# For each extra ASL_field, it has a pair of 8-byte fields where the first
# 8 bytes contains the name of the extra ASL_field and the second 8 bytes
# contains the text of the extra field.
# All of this 8-byte field can be saved using one of these three different
# types:
# - Null value ('0000000000000000'): nothing to do.
# - String: It is string if first bit = 1 or first nibble = 8 (1000).
# Second nibble has the length of string.
# The next 7 bytes have the text characters of the string
# padding the end with null characters: '0x00'.
# Example: [8468 6964 6400 0000]
# [8] String, [4] length, value: [68 69 64 64] = hidd.
# - Pointer: static position in the file to a special struct
# implemented as an ASL_RECORD_DYN_VALUE.
# Example: [0000 0000 0000 0077]
# It points to the file position 0x077 that has a
# ASL_RECORD_DYN_VALUE structure.
values = []
while tam_fields > 0:
try:
field_data = file_object.read(8)
except IOError as exception:
parser_mediator.ProduceParseError(
u'unable to read ASL field with error: {0:s}'.format(
exception))
return None, None
# Try to read the field data as a string.
try:
asl_string_struct = self._ASL_STRING.parse(field_data)
string_data = b''.join(
asl_string_struct.string[0:asl_string_struct.
string_length])
values.append(string_data)
# Go to parse the next extra field.
tam_fields -= 8
continue
except ValueError:
pass
# If the field is not a string it must be a pointer.
try:
pointer_value = self._ASL_POINTER.parse(field_data)
except ValueError as exception:
parser_mediator.ProduceParseError(
u'unable to parse ASL field with error: {0:s}'.format(
exception))
return None, None
if not pointer_value:
# Next extra field: 8 bytes more.
tam_fields -= 8
continue
# The next IF ELSE is only for performance issues, avoiding seek.
# If the pointer points a lower position than where the actual entry
# starts, it means that it points to a previous entry.
pos = pointer_value - dynamic_data_offset
# Greater or equal 0 means that the data is in the actual entry.
if pos >= 0:
try:
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse(
dynamic_data[pos:])
dyn_value = dyn_value_struct.value.partition(b'\x00')[0]
values.append(dyn_value)
except (IOError, construct.FieldError) as exception:
parser_mediator.ProduceParseError((
u'unable to parse ASL record dynamic value with error: '
u'{0:s}').format(exception))
return None, None
else:
# Only if it is a pointer that points to the
# heap from another entry we use the seek method.
main_position = file_object.tell()
# If the pointer is in a previous entry.
if main_position > pointer_value:
file_object.seek(pointer_value - main_position,
os.SEEK_CUR)
try:
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream(
file_object)
dyn_value = dyn_value_struct.value.partition(
b'\x00')[0]
values.append(dyn_value)
except (IOError, construct.FieldError):
parser_mediator.ProduceParseError((
u'the pointer at {0:d} (0x{0:08x}) points to invalid '
u'information.'
).format(main_position - self._ASL_POINTER.sizeof()))
# Come back to the position in the entry.
_ = file_object.read(main_position - file_object.tell())
else:
_ = file_object.read(pointer_value - main_position)
dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream(
file_object)
dyn_value = dyn_value_struct.value.partition(b'\x00')[0]
values.append(dyn_value)
# Come back to the position in the entry.
file_object.seek(main_position - file_object.tell(),
os.SEEK_CUR)
# Next extra field: 8 bytes more.
tam_fields -= 8
# Read the last 8 bytes of the record that points to the previous entry.
_ = file_object.read(8)
# Parsed section, we translate the read data to an appropriate format.
micro_seconds, _ = divmod(record_struct.nanosec, 1000)
# Parsing the dynamic values (text or pointers to position with text).
# The first four are always the host, sender, facility, and message.
number_of_values = len(values)
if number_of_values < 4:
parser_mediator.ProduceParseError(
u'less than four values read from an ASL event.')
computer_name = u'N/A'
sender = u'N/A'
facility = u'N/A'
message = u'N/A'
if number_of_values >= 1:
computer_name = values[0].decode(u'utf-8')
if number_of_values >= 2:
sender = values[1].decode(u'utf-8')
if number_of_values >= 3:
facility = values[2].decode(u'utf-8')
if number_of_values >= 4:
message = values[3].decode(u'utf-8')
# If the entry has an extra fields, they works as a pairs:
# The first is the name of the field and the second the value.
extra_information = u''
if number_of_values > 4 and number_of_values % 2 == 0:
# Taking all the extra attributes and merging them together,
# eg: a = [1, 2, 3, 4] will look like "1: 2, 3: 4".
try:
extra_values = map(py2to3.UNICODE_TYPE, values[4:])
extra_information = u', '.join(
map(u': '.join, zip(extra_values[0::2],
extra_values[1::2])))
except UnicodeDecodeError as exception:
parser_mediator.ProduceParseError(
u'Unable to decode all ASL values in the extra information fields.'
)
event_object = ASLEvent(record_struct.timestamp,
offset,
record_struct.asl_message_id,
record_struct.level,
record_struct.pid,
record_struct.uid,
record_struct.gid,
record_struct.read_uid,
record_struct.read_gid,
computer_name,
sender,
facility,
message,
extra_information,
micro_seconds=micro_seconds)
parser_mediator.ProduceEvent(event_object)
return (event_object, record_struct.next_offset)
tx_core_name = 'gbe0'
test_core_bram = 'qdrBram'
test_QDRReg = 'testQDR'
test_fineFFT = 'vals_testQDR'
fpga = []
snap_debug = 'snap_debug'
snap_fengine_debug_coarse_fft = construct.BitStruct(
snap_debug, construct.Padding(128 - (4 * 18)),
construct.BitField("d0_r", 18), construct.BitField("d0_i", 18),
construct.BitField("d1_r", 18), construct.BitField("d1_i", 18))
# the xaui snap block on the f-engine - this is just after packetisation
snap_fengine_xaui = construct.BitStruct(
"snap_debug", construct.Padding(128 - 1 - 3 - 1 - 1 - 3 - 64),
construct.Flag("link_down"), construct.Padding(3), construct.Flag("mrst"),
construct.Padding(1), construct.Flag("eof"), construct.Flag("sync"),
construct.Flag("hdr_valid"), construct.BitField("data", 64))
def bin2fp(bits, m=8, e=7):
if m > 32:
raise RuntimeError('Unsupported fixed format: %i.%i' % (m, e))
shift = 32 - m
bits = bits << shift
m = m + shift
e = e + shift
return float(numpy.int32(bits)) / (2**e)
def bram(bname, odata, samples=1024):
import construct, corr_functions
register_fengine_control = construct.BitStruct(
'feng_ctl',
construct.Flag('gbe_gpu_rst'), #31
construct.Flag('gbe_sw_rst'), #30
construct.Flag('loopback_mux_rst'), #29
construct.Flag('cnt_rst'), #28
construct.BitField('fft_preshift', 2), #26-27
construct.BitField('gpio_monsel', 3), #23,24,25
construct.Flag('fft_tvg2'), #22
construct.Flag('fft_tvg1'), #21
construct.Flag('gbe_gpu_disable'), #20
construct.Flag('use_qdr_tvg'), #19
construct.Flag('gbe_sw_disable'), #18
construct.Flag('arm_rst'), #17
construct.Flag('sync_rst'), #16
construct.Flag('arm_noise'), #15
construct.Padding(12), #3-14
construct.Flag('lb_err_cnt_rst'), #2
construct.Padding(2)) #0-1
# from . import snap, corr_functions
def bin2fp(bits, m = 8, e = 7):
if m > 32:
raise RuntimeError('Unsupported fixed format: %i.%i' % (m,e))
shift = 32 - m
bits = bits << shift
m = m + shift
e = e + shift
return float(numpy.int32(bits)) / (2**e)
if construct.version[1] <= 6:
# f-engine adc control
register_fengine_adc_control = construct.BitStruct('adc_ctrl0',
construct.Flag('enable'), # 31 Enable input channel on KAT ADC.
construct.Padding(32 - 6 - 1), # 6-30
construct.BitField('atten', 6)) # 0-5 KAT ADC channel attenuation setting.
# f-engine status
register_fengine_fstatus = construct.BitStruct('fstatus0',
construct.BitField('coarse_bits', 5), # 27-31 2^x - the number of points in the coarse FFT.
construct.BitField('fine_bits', 5), # 22-26 2^y - the number of points in the fine FFT.
construct.BitField('sync_val', 2), # 20-21 On which ADC cycle did the sync happen?
construct.Padding(2), # 18-19
construct.Flag('xaui_lnkdn'), # 17 The 10GBE link is down.
construct.Flag('xaui_over'), # 16 The 10GBE link has overflows.
construct.Padding(9), # 7-15
construct.Flag('clk_err'), # 6 The board frequency is calculated out of bounds.
construct.Flag('adc_disabled'), # 5 The ADC has been disabled.
construct.Flag('ct_error'), # 4 There is a QDR error from the corner-turner.
class FlacReader:
FRAME_HEADER = construct.Struct(
'frame_header', construct.Bits('sync', 14),
construct.Bits('reserved', 2), construct.Bits('block_size', 4),
construct.Bits('sample_rate', 4),
construct.Bits('channel_assignment', 4),
construct.Bits('bits_per_sample', 3), construct.Padding(1),
construct.IfThenElse(
'total_channels', lambda ctx1: ctx1['channel_assignment'] <= 7,
construct.Value('c', lambda ctx2: ctx2['channel_assignment'] + 1),
construct.Value('c', lambda ctx3: 2)), UTF8('frame_number'),
construct.IfThenElse(
'extended_block_size', lambda ctx1: ctx1['block_size'] == 6,
construct.Bits('b', 8),
construct.If(lambda ctx2: ctx2['block_size'] == 7,
construct.Bits('b', 16))),
construct.IfThenElse(
'extended_sample_rate', lambda ctx1: ctx1['sample_rate'] == 12,
construct.Bits('s', 8),
construct.If(lambda ctx2: ctx2['sample_rate'] in (13, 14),
construct.Bits('s', 16))), construct.Bits('crc8', 8))
UNARY = construct.Struct(
'unary',
construct.RepeatUntil(lambda obj, ctx: obj == '\x01',
construct.Field('bytes', 1)),
construct.Value('value', lambda ctx: len(ctx['bytes']) - 1))
SUBFRAME_HEADER = construct.Struct(
'subframe_header', construct.Padding(1),
construct.Bits('subframe_type', 6),
construct.Flag('has_wasted_bits_per_sample'),
construct.IfThenElse('wasted_bits_per_sample',
lambda ctx: ctx['has_wasted_bits_per_sample'],
PlusOne(Unary('value')),
construct.Value('value', lambda ctx2: 0)))
GET_BLOCKSIZE_FROM_STREAMINFO = -1
GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER = -2
GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER = -3
BLOCK_SIZE = (GET_BLOCKSIZE_FROM_STREAMINFO, 192, 576, 1152, 2304, 4608,
GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER,
GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER, 256, 512, 1024, 2048,
4096, 8192, 16384, 32768)
GET_SAMPLE_SIZE_FROM_STREAMINFO = -1
SAMPLE_SIZE = (GET_SAMPLE_SIZE_FROM_STREAMINFO, 8, 12, None, 16, 20, 24,
None)
def FIXED0(subframe, residual, i):
subframe.insert(i, residual[i])
def FIXED1(subframe, residual, i):
subframe.insert(i, subframe[i - 1] + residual[i])
def FIXED2(subframe, residual, i):
subframe.insert(i,
((2 * subframe[i - 1]) - subframe[i - 2] + \
residual[i]))
def FIXED3(subframe, residual, i):
subframe.insert(i,
((3 * subframe[i - 1]) - (3 * subframe[i - 2]) + \
subframe[i - 3] + residual[i]))
def FIXED4(subframe, residual, i):
subframe.insert(i,
((4 * subframe[i - 1]) - (6 * subframe[i - 2]) + \
(4 * subframe[i - 3]) - subframe[i - 4] + residual[i]))
#iterates over all of the channels, in order
def MERGE_INDEPENDENT(channel_list):
channel_data = [iter(c) for c in channel_list]
while (True):
for channel in channel_data:
yield channel.next()
def MERGE_LEFT(channel_list):
channel_left = iter(channel_list[0])
channel_side = iter(channel_list[1])
while (True):
left = channel_left.next()
side = channel_side.next()
yield left
yield left - side
def MERGE_RIGHT(channel_list):
channel_side = iter(channel_list[0])
channel_right = iter(channel_list[1])
while (True):
side = channel_side.next()
right = channel_right.next()
yield side + right
yield right
def MERGE_MID(channel_list):
channel_mid = iter(channel_list[0])
channel_side = iter(channel_list[1])
while (True):
mid = channel_mid.next()
side = channel_side.next()
mid = mid << 1
mid |= (side & 0x1)
yield (mid + side) >> 1
yield (mid - side) >> 1
CHANNEL_FUNCTIONS = (MERGE_INDEPENDENT, MERGE_INDEPENDENT,
MERGE_INDEPENDENT, MERGE_INDEPENDENT,
MERGE_INDEPENDENT, MERGE_INDEPENDENT,
MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_LEFT,
MERGE_RIGHT, MERGE_MID)
FIXED_FUNCTIONS = (FIXED0, FIXED1, FIXED2, FIXED3, FIXED4)
def __init__(self, flac_stream):
self.stream = BufferedStream(flac_stream)
self.streaminfo = None
self.bitstream = None
#ensure the file starts with 'fLaC'
self.read_stream_marker()
#initialize self.bitstream
self.begin_bitstream()
#find self.streaminfo in case we need it
self.read_metadata_blocks()
def close(self):
if (self.bitstream != None):
self.bitstream.close()
else:
self.stream.close()
def read_stream_marker(self):
if (self.stream.read(4) != 'fLaC'):
raise FlacStreamException('invalid stream marker')
def read_metadata_blocks(self):
block = audiotools.FlacAudio.METADATA_BLOCK_HEADER.parse_stream(
self.stream)
while (block.last_block == 0):
if (block.block_type == 0):
self.streaminfo = audiotools.FlacAudio.STREAMINFO.parse_stream(
self.stream)
else:
self.stream.seek(block.block_length, 1)
block = audiotools.FlacAudio.METADATA_BLOCK_HEADER.parse_stream(
self.stream)
self.stream.seek(block.block_length, 1)
def begin_bitstream(self):
import bitstream
#self.bitstream = construct.BitStreamReader(self.stream)
self.bitstream = bitstream.BitStreamReader(self.stream)
def read_frame(self):
self.stream.reset_buffer()
try:
header = FlacReader.FRAME_HEADER.parse_stream(self.bitstream)
except construct.core.FieldError:
return ""
if (header.sync != 0x3FFE):
raise FlacStreamException('invalid sync')
if (crc8(self.stream.getvalue()[0:-1]) != header.crc8):
raise FlacStreamException('crc8 checksum failed')
#block_size tells us how many samples we need from each subframe
block_size = FlacReader.BLOCK_SIZE[header.block_size]
if (block_size == self.GET_BLOCKSIZE_FROM_STREAMINFO):
block_size = self.streaminfo.maximum_blocksize
elif ((block_size == self.GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER)
or (block_size == self.GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER)):
block_size = header.extended_block_size + 1
#grab subframe data as 32-bit array objects
subframe_data = []
for channel_number in xrange(header.total_channels):
subframe_data.append(
self.read_subframe(header, block_size, channel_number))
crc16sum = crc16(self.stream.getvalue())
#try to byte-align the stream
if (len(self.bitstream.buffer) > 0):
self.bitstream.read(len(self.bitstream.buffer))
if (crc16sum != construct.Bits('crc16', 16).parse_stream(
self.bitstream)):
raise FlacStreamException('crc16 checksum failed')
#convert our list of subframe data arrays into
#a string of sample data
if (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == 16):
merged_frames = array.array(
'h', FlacReader.CHANNEL_FUNCTIONS[header.channel_assignment](
subframe_data))
if (audiotools.BIG_ENDIAN):
merged_frames.byteswap()
return merged_frames.tostring()
elif (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == 8):
merged_frames = array.array(
'b', FlacReader.CHANNEL_FUNCTIONS[header.channel_assignment](
subframe_data))
return merged_frames.tostring()
else:
if (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == \
self.GET_SAMPLE_SIZE_FROM_STREAMINFO):
bits_per_sample = self.streaminfo.bits_per_sample + 1
elif (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == None):
raise FlacStreamException('invalid bits per sample')
else:
bits_per_sample = FlacReader.SAMPLE_SIZE[
header.bits_per_sample]
stream = construct.GreedyRepeater(
construct.BitStruct(
'bits',
construct.Bits('value',
bits_per_sample,
swapped=True,
signed=True)))
return stream.build([
construct.Container(value=v)
for v in FlacReader.CHANNEL_FUNCTIONS[
header.channel_assignment](subframe_data)
])
def read_subframe(self, frame_header, block_size, channel_number):
subframe_header = \
FlacReader.SUBFRAME_HEADER.parse_stream(self.bitstream)
#figure out the bits-per-sample of this subframe
if ((frame_header.channel_assignment == 8) and (channel_number == 1)):
#if channel is stored as left+difference
#and this is the difference, add 1 bit
bits_per_sample = FlacReader.SAMPLE_SIZE[
frame_header.bits_per_sample] + 1
elif ((frame_header.channel_assignment == 9)
and (channel_number == 0)):
#if channel is stored as difference+right
#and this is the difference, add 1 bit
bits_per_sample = FlacReader.SAMPLE_SIZE[
frame_header.bits_per_sample] + 1
elif ((frame_header.channel_assignment == 10)
and (channel_number == 1)):
#if channel is stored as average+difference
#and this is the difference, add 1 bit
bits_per_sample = FlacReader.SAMPLE_SIZE[
frame_header.bits_per_sample] + 1
else:
#otherwise, use the number from the frame header
bits_per_sample = FlacReader.SAMPLE_SIZE[
frame_header.bits_per_sample]
if (subframe_header.has_wasted_bits_per_sample):
bits_per_sample -= subframe_header.wasted_bits_per_sample
if (subframe_header.subframe_type == 0):
subframe = self.read_subframe_constant(block_size, bits_per_sample)
elif (subframe_header.subframe_type == 1):
subframe = self.read_subframe_verbatim(block_size, bits_per_sample)
elif ((subframe_header.subframe_type & 0x38) == 0x08):
subframe = self.read_subframe_fixed(
subframe_header.subframe_type & 0x07, block_size,
bits_per_sample)
elif ((subframe_header.subframe_type & 0x20) == 0x20):
subframe = self.read_subframe_lpc(
(subframe_header.subframe_type & 0x1F) + 1, block_size,
bits_per_sample)
else:
raise FlacStreamException('invalid subframe type')
if (subframe_header.has_wasted_bits_per_sample):
return array.array('i', [
i << subframe_header.wasted_bits_per_sample for i in subframe
])
else:
return subframe
def read_subframe_constant(self, block_size, bits_per_sample):
sample = construct.Bits('b',
bits_per_sample).parse_stream(self.bitstream)
subframe = array.array('i', [sample] * block_size)
return subframe
def read_subframe_verbatim(self, block_size, bits_per_sample):
return array.array(
'i',
construct.StrictRepeater(
block_size,
construct.Bits("samples", bits_per_sample,
signed=True)).parse_stream(self.bitstream))
def read_subframe_fixed(self, order, block_size, bits_per_sample):
samples = construct.StrictRepeater(
order,
construct.Bits("warm_up_samples", bits_per_sample, signed=True))
subframe = array.array('i', samples.parse_stream(self.bitstream))
residual = self.read_residual(block_size, order)
fixed_func = self.FIXED_FUNCTIONS[order]
for i in xrange(len(subframe), block_size):
fixed_func(subframe, residual, i)
return subframe
def read_subframe_lpc(self, order, block_size, bits_per_sample):
samples = construct.StrictRepeater(
order,
construct.Bits("warm_up_samples", bits_per_sample, signed=True))
subframe = array.array('i', samples.parse_stream(self.bitstream))
lpc_precision = construct.Bits('lpc_precision', 4).parse_stream(
self.bitstream) + 1
lpc_shift = construct.Bits('lpc_shift', 5).parse_stream(self.bitstream)
coefficients = array.array(
'i',
construct.StrictRepeater(
order,
construct.Bits('coefficients', lpc_precision,
signed=True)).parse_stream(self.bitstream))
residual = self.read_residual(block_size, order)
for i in xrange(len(subframe), block_size):
subframe.insert(i,
(sum(
[coefficients[j] * subframe[i - j - 1] for j in
xrange(0,len(coefficients))]) >> lpc_shift) + \
residual[i])
return subframe
def read_residual(self, block_size, predictor_order):
rice = array.array('i')
#add some dummy rice so that the Rice index matches
#that of the rest of the subframe
for i in xrange(predictor_order):
rice.append(0)
coding_method = self.bitstream.read(2)
if (coding_method == '\x00\x00'):
rice2 = False
elif (coding_method == '\x00\x01'):
rice2 = True
else:
raise FlacStreamException('invalid residual coding method')
partition_order = construct.Bits('partition_order',
4).parse_stream(self.bitstream)
if (partition_order > 0):
total_samples = ((block_size / 2**partition_order) -
predictor_order)
rice.extend(self.read_encoded_rice(total_samples, rice2))
for i in xrange(1, 2**partition_order):
total_samples = (block_size / 2**partition_order)
rice.extend(self.read_encoded_rice(total_samples, rice2))
else:
rice.extend(
self.read_encoded_rice(block_size - predictor_order, rice2))
return rice
def read_encoded_rice(self, total_samples, rice2=False):
bin_to_int = construct.lib.binary.bin_to_int
samples = array.array('i')
if (not rice2):
rice_parameter = construct.Bits('rice_parameter',
4).parse_stream(self.bitstream)
else:
rice_parameter = construct.Bits('rice_parameter',
5).parse_stream(self.bitstream)
if (rice_parameter != 0xF):
#a Rice encoded residual
for x in xrange(total_samples):
#count the number of 0 bits before the next 1 bit
#(unary encoding)
#to find our most significant bits
msb = 0
s = self.bitstream.read(1)
while (s != '\x01'):
msb += 1
s = self.bitstream.read(1)
#grab the proper number of least significant bits
lsb = bin_to_int(self.bitstream.read(rice_parameter))
#combine msb and lsb to get the Rice-encoded value
value = (msb << rice_parameter) | lsb
if ((value & 0x1) == 0x1): #negative
samples.append(-(value >> 1) - 1)
else: #positive
samples.append(value >> 1)
else:
#unencoded residual
bits_per_sample = construct.Bits('escape_code',
5).parse_stream(self.bitstream)
sample = construct.Bits("sample", bits_per_sample, signed=True)
for x in xrange(total_samples):
samples.append(sample.parse_stream(self.bitstream))
return samples
fine_fft_size = 4096 # Frequency bins
offset_shift_size = snap_size # For the fine FFT, when the whole thing can't be measured at once.
num_offset_shifts = 4 # In case this later needs to be changed, perhaps for a different size fine FFT.
# Variables relevant only to the corner turner
snaps_per_spectrum = 128
channels_per_snap = 32
words_per_channel = 32
# Bitstruct to control the control register on the ROACH
control_reg_bitstruct = construct.BitStruct(
'control_reg',
construct.Padding(4), # 28-31
construct.BitField('debug_snap_select', 3), # 25-27
construct.Padding(3), # 22-24
construct.Flag('fine_tvg_en'), # 21
construct.Flag('adc_tvg'), # 20
construct.Flag('fd_fs_tvg'), # 19
construct.Flag('packetiser_tvg'), # 18
construct.Flag('ct_tvg'), # 17
construct.Flag('tvg_en'), # 16
construct.Padding(4), # 12-15
construct.Flag('fancy_en'), # 11
construct.Flag('adc_protect_disable'), # 10
construct.Flag('gbe_enable'), # 09
construct.Flag('gbe_rst'), # 08
construct.Padding(4), # 04-07
construct.Flag('clr_status'), # 03
construct.Flag('arm'), # 02
construct.Flag('man_sync'), # 01
construct.Flag('sys_rst')) # 00
