def __init__(self): super(VideoHandler, self).__init__() self.last_sent_time = 0 self.decoder = H264Decoder() self.header = construct.BitStruct( 'VSTRMHeader', construct.Nibble('magic'), construct.BitField('packet_type', 2), construct.BitField('seq_id', 10), construct.Flag('init'), construct.Flag('frame_begin'), construct.Flag('chunk_end'), construct.Flag('frame_end'), construct.Flag('has_timestamp'), construct.BitField('payload_size', 11), construct.BitField('timestamp', 32)) self.frame = array.array('B') self.is_streaming = False self.frame_decode_num = 0
def __init__(self): super(AudioHandler, self).__init__() self.header_base = construct.BitStruct( 'ASTRMBaseHeader', construct.BitField('fmt', 3), construct.Bit('channel'), construct.Flag('vibrate'), construct.Bit('packet_type'), construct.BitField('seq_id', 10), construct.BitField('payload_size', 16)) self.header_aud = construct.Struct('ASTRMAudioHeader', construct.ULInt32('timestamp')) self.header_msg = construct.Struct( 'ASTRMMsgHeader', # This is kind of a hack, (there are two timestamp fields, which one is used # depends on packet_type construct.ULInt32('timestamp_audio'), construct.ULInt32('timestamp'), construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync construct.ULInt8('vid_format'), construct.Padding(3)) self.header = construct.Struct( 'ASTRMHeader', construct.Embed(self.header_base), construct.Switch('format_hdr', lambda ctx: ctx.packet_type, { 0: construct.Embed(self.header_aud), 1: construct.Embed(self.header_msg), }, default=construct.Pass))
def __init__(s): super(ServiceVSTRM, s).__init__() s.decoder = H264Decoder(s.dimensions['gamepad'], pygame.display.get_surface().get_size()) s.header = construct.BitStruct('VSTRMHeader', construct.Nibble('magic'), construct.BitField('packet_type', 2), construct.BitField('seq_id', 10), construct.Flag('init'), construct.Flag('frame_begin'), construct.Flag('chunk_end'), construct.Flag('frame_end'), construct.Flag('has_timestamp'), construct.BitField('payload_size', 11), construct.BitField('timestamp', 32)) s.frame = array.array('B') s.is_streaming = False s.frame_decode_num = 0
def __init__(s, decoder): super(ServiceVSTRM, s).__init__() s.decoder = decoder s.header = construct.BitStruct('VSTRMHeader', construct.Nibble('magic'), construct.BitField('packet_type', 2), construct.BitField('seq_id', 10), construct.Flag('init'), construct.Flag('frame_begin'), construct.Flag('chunk_end'), construct.Flag('frame_end'), construct.Flag('has_timestamp'), construct.BitField('payload_size', 11), construct.BitField('timestamp', 32)) s.frame = array.array('B') s._is_streaming = False s.is_streaming = False s.frame_decode_num = 0
def __init__(s): super(ServiceASTRM, s).__init__() s.header_base = construct.BitStruct('ASTRMBaseHeader', construct.BitField('fmt', 3), construct.Bit('channel'), construct.Flag('vibrate'), construct.Bit('packet_type'), construct.BitField('seq_id', 10), construct.BitField('payload_size', 16) ) s.header_aud = construct.Struct('ASTRMAudioHeader', construct.ULInt32('timestamp'), # construct.Array(lambda ctx: ctx.payload_size, construct.UBInt8("data")) ) s.header_msg = construct.Struct('ASTRMMsgHeader', # This is kind of a hack, (there are two timestamp fields, which one is used depends on packet_type construct.ULInt32('timestamp_audio'), construct.ULInt32('timestamp'), construct.Array(2, construct.ULInt32('freq_0')), # -> mc_video construct.Array(2, construct.ULInt32('freq_1')), # -> mc_sync construct.ULInt8('vid_format'), construct.Padding(3) ) s.header = construct.Struct('ASTRMHeader', construct.Embed(s.header_base), construct.Switch('format_hdr', lambda ctx: ctx.packet_type, { 0 : construct.Embed(s.header_aud), 1 : construct.Embed(s.header_msg), }, default = construct.Pass ) ) s.is_streaming = False s.p = pyaudio.PyAudio() s.stream = None s.pa_num_bufs = 15 s.pa_ring = [array.array('H', '\0' * 416 * 2)] * s.pa_num_bufs s.pa_wpos = s.pa_rpos = 0
""" Setup and unique functionality for the wide-band correlator modes. A wideband correlator's FPGAs process all digitised data, which is a multiple of the FPGA clock rates. """ """ Revisions: 2011-07-07 PVP Initial revision. """ import construct, corr_functions # f-engine control register register_fengine_control = construct.BitStruct( 'control', construct.Padding(32 - 20 - 1), # 21 - 31 construct.Flag('tvgsel_noise'), # 20 construct.Flag('tvgsel_fdfs'), # 19 construct.Flag('tvgsel_pkt'), # 18 construct.Flag('tvgsel_ct'), # 17 construct.Flag('tvg_en'), # 16 construct.Padding(16 - 13 - 1), # 14 - 15 construct.Flag('adc_protect_disable'), # 13 construct.Flag('flasher_en'), # 12 construct.Padding(12 - 9 - 1), # 10 - 11 construct.Flag('gbe_enable'), # 9 construct.Flag('gbe_rst'), # 8 construct.Padding(8 - 3 - 1), # 4 - 7 construct.Flag('clr_status'), # 3 construct.Flag('arm'), # 2 construct.Flag('soft_sync'), # 1 construct.Flag('mrst')) # 0 # f-engine status
import construct as cst import time import sys con = cst.Container # alias HEADER = cst.Struct('pc_header', cst.Magic('\x00'), cst.Const(cst.UBInt8('lines'), 1), cst.UBInt8('address'), cst.Magic('\x03')) SER_STATUS = cst.BitStruct('serst', cst.Magic('\x01\x01\x00'), cst.Flag('schedule_enabled'), cst.Flag('ack_enabled'), cst.Flag('further_pages'), cst.Flag('interrupt_mode'), cst.Magic('\x00')) PAGE_IDX = cst.Bytes('page_num', 3) TEMPO = cst.BitStruct( 'tempo', cst.Magic('\x01\x01'), cst.Enum(cst.BitField('display_ctrl', 2), TIMED=0, FIXED_ON=1, FIXED_OFF=2), cst.Enum(cst.BitField('persist_time', 4), S2=1, S5=2, S10=3, S20=4, S30=5, S45=6, S60=7, S90=8, S120=9))
class AslParser(interface.BaseParser): """Parser for ASL log files.""" NAME = 'asl_log' DESCRIPTION = u'Parser for ASL log files.' ASL_MAGIC = 'ASL DB\x00\x00\x00\x00\x00\x00' # If not right assigned, the value is "-1". ASL_NO_RIGHTS = 'ffffffff' # Priority level (criticity) ASL_MESSAGE_PRIORITY = { 0: 'EMERGENCY', 1: 'ALERT', 2: 'CRITICAL', 3: 'ERROR', 4: 'WARNING', 5: 'NOTICE', 6: 'INFO', 7: 'DEBUG' } # ASL File header. # magic: magic number that identify ASL files. # version: version of the file. # offset: first record in the file. # timestamp: epoch time when the first entry was written. # last_offset: last record in the file. ASL_HEADER_STRUCT = construct.Struct('asl_header_struct', construct.String('magic', 12), construct.UBInt32('version'), construct.UBInt64('offset'), construct.UBInt64('timestamp'), construct.UBInt32('cache_size'), construct.UBInt64('last_offset'), construct.Padding(36)) # The record structure is: # [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry] # Record static structure. # tam_entry: it contains the number of bytes from this file position # until the end of the record, without counts itself. # next_offset: next record. If is equal to 0x00, it is the last record. # asl_message_id: integer that has the numeric identification of the event. # timestamp: Epoch integer that has the time when the entry was created. # nanosecond: nanosecond to add to the timestamp. # level: level of priority. # pid: process identification that ask to save the record. # uid: user identification that has lunched the process. # gid: group identification that has lunched the process. # read_uid: identification id of a user. Only applied if is not -1 (all FF). # Only root and this user can read the entry. # read_gid: the same than read_uid, but for the group. ASL_RECORD_STRUCT = construct.Struct('asl_record_struct', construct.Padding(2), construct.UBInt32('tam_entry'), construct.UBInt64('next_offset'), construct.UBInt64('asl_message_id'), construct.UBInt64('timestamp'), construct.UBInt32('nanosec'), construct.UBInt16('level'), construct.UBInt16('flags'), construct.UBInt32('pid'), construct.UBInt32('uid'), construct.UBInt32('gid'), construct.UBInt32('read_uid'), construct.UBInt32('read_gid'), construct.UBInt64('ref_pid')) ASL_RECORD_STRUCT_SIZE = ASL_RECORD_STRUCT.sizeof() # 8-byte fields, they can be: # - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String]. # - Integer: integer that has the byte position in the file that points # to an ASL_RECORD_DYN_VALUE struct. If the value of the integer # is equal to 0, it means that it has not data (skip). # If the field is a String, we use this structure to decode each # integer byte in the corresponding character (ASCII Char). ASL_OCTET_STRING = construct.ExprAdapter(construct.Octet('string'), encoder=lambda obj, ctx: ord(obj), decoder=lambda obj, ctx: chr(obj)) # Field string structure. If the first bit is 1, it means that it # is a String (1000) = 8, then the next nibble has the number of # characters. The last 7 bytes are the number of bytes. ASL_STRING = construct.BitStruct( 'string', construct.Flag('type'), construct.Bits('filler', 3), construct.If(lambda ctx: ctx.type, construct.Nibble('string_length')), construct.If(lambda ctx: ctx.type, construct.Array(7, ASL_OCTET_STRING))) # 8-byte pointer to a byte position in the file. ASL_POINTER = construct.UBInt64('pointer') # Dynamic data structure pointed by a pointer that contains a String: # [2 bytes padding][4 bytes lenght of String][String]. ASL_RECORD_DYN_VALUE = construct.Struct( 'asl_record_dyn_value', construct.Padding(2), construct.PascalString('value', length_field=construct.UBInt32('length'))) def Parse(self, parser_context, file_entry): """Extract entries from an ASL file. Args: parser_context: A parser context object (instance of ParserContext). file_entry: A file entry object (instance of dfvfs.FileEntry). """ file_object = file_entry.GetFileObject() file_object.seek(0, os.SEEK_SET) try: header = self.ASL_HEADER_STRUCT.parse_stream(file_object) except (IOError, construct.FieldError) as exception: file_object.close() raise errors.UnableToParseFile( u'Unable to parse ASL Header with error: {0:s}.'.format( exception)) if header.magic != self.ASL_MAGIC: file_object.close() raise errors.UnableToParseFile( u'Not an ASL Header, unable to parse.') # Get the first and the last entry. offset = header.offset old_offset = header.offset last_offset_header = header.last_offset # If the ASL file has entries. if offset: event_object, offset = self.ReadAslEvent(file_object, offset) while event_object: parser_context.ProduceEvent(event_object, parser_name=self.NAME, file_entry=file_entry) # TODO: an anomaly object must be emitted once that is implemented. # Sanity check, the last read element must be the same as # indicated by the header. if offset == 0 and old_offset != last_offset_header: logging.warning(u'Parsing ended before the header ends.') old_offset = offset event_object, offset = self.ReadAslEvent(file_object, offset) file_object.close() def ReadAslEvent(self, file_object, offset): """Returns an AslEvent from a single ASL entry. Args: file_object: a file-like object that points to an ASL file. offset: offset where the static part of the entry starts. Returns: An event object constructed from a single ASL record. """ # The heap of the entry is saved to try to avoid seek (performance issue). # It has the real start position of the entry. dynamic_start = file_object.tell() dynamic_part = file_object.read(offset - file_object.tell()) if not offset: return None, None try: record_header = self.ASL_RECORD_STRUCT.parse_stream(file_object) except (IOError, construct.FieldError) as exception: logging.warning( u'Unable to parse ASL event with error: {0:s}'.format( exception)) return None, None # Variable tam_fields = is the real length of the dynamic fields. # We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before] # In Record_Struct we have a field called tam_entry, where it has the number # of bytes until the end of the entry from the position that the field is. # The tam_entry is between the 2th and the 6th byte in the [Record_Struct]. # tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before] # Also, we do not need [Point_Entry_Before] and then we delete the size of # [Point_Entry_Before] that it is 8 bytes (8): # tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before] # [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8 # [Dynamic_Fields] = tam_entry - [Record_Struct] - 2 tam_fields = record_header.tam_entry - self.ASL_RECORD_STRUCT_SIZE - 2 # Dynamic part of the entry that contains minimal four fields of 8 bytes # plus 2x[8bytes] fields for each extra ASL_Field. # The four first fields are always the Host, Sender, Facility and Message. # After the four first fields, the entry might have extra ASL_Fields. # For each extra ASL_field, it has a pair of 8-byte fields where the first # 8 bytes contains the name of the extra ASL_field and the second 8 bytes # contains the text of the exta field. # All of this 8-byte field can be saved using one of these three differents # types: # - Null value ('0000000000000000'): nothing to do. # - String: It is string if first bit = 1 or first nibble = 8 (1000). # Second nibble has the length of string. # The next 7 bytes have the text characters of the string # padding the end with null characters: '0x00'. # Example: [8468 6964 6400 0000] # [8] String, [4] length, value: [68 69 64 64] = hidd. # - Pointer: static position in the file to a special struct # implemented as an ASL_RECORD_DYN_VALUE. # Example: [0000 0000 0000 0077] # It points to the file position 0x077 that has a # ASL_RECORD_DYN_VALUE structure. values = [] while tam_fields > 0: try: raw_field = file_object.read(8) except (IOError, construct.FieldError) as exception: logging.warning( u'Unable to parse ASL event with error: {0:d}'.format( exception)) return None, None try: # Try to read as a String. field = self.ASL_STRING.parse(raw_field) values.append(''.join(field.string[0:field.string_length])) # Go to parse the next extra field. tam_fields -= 8 continue except ValueError: pass # If it is not a string, it must be a pointer. try: field = self.ASL_POINTER.parse(raw_field) except ValueError as exception: logging.warning( u'Unable to parse ASL event with error: {0:s}'.format( exception)) return None, None if field != 0: # The next IF ELSE is only for performance issues, avoiding seek. # If the pointer points a lower position than where the actual entry # starts, it means that it points to a previuos entry. pos = field - dynamic_start # Bigger or equal 0 means that the data is in the actual entry. if pos >= 0: try: values.append((self.ASL_RECORD_DYN_VALUE.parse( dynamic_part[pos:])).value.partition('\x00')[0]) except (IOError, construct.FieldError) as exception: logging.warning( u'Unable to parse ASL event with error: {0:s}'. format(exception)) return None, None else: # Only if it is a pointer that points to the # heap from another entry we use the seek method. main_position = file_object.tell() # If the pointer is in a previous entry. if main_position > field: file_object.seek(field - main_position, os.SEEK_CUR) try: values.append( (self.ASL_RECORD_DYN_VALUE.parse_stream( file_object)).value.partition('\x00')[0]) except (IOError, construct.FieldError): logging.warning(( u'The pointer at {0:d} (0x{0:x}) points to invalid ' u'information.' ).format(main_position - self.ASL_POINTER.sizeof())) # Come back to the position in the entry. _ = file_object.read(main_position - file_object.tell()) else: _ = file_object.read(field - main_position) values.append((self.ASL_RECORD_DYN_VALUE.parse_stream( file_object)).value.partition('\x00')[0]) # Come back to the position in the entry. file_object.seek(main_position - file_object.tell(), os.SEEK_CUR) # Next extra field: 8 bytes more. tam_fields -= 8 # Read the last 8 bytes of the record that points to the previous entry. _ = file_object.read(8) # Parsed section, we translate the read data to an appropriate format. microsecond = record_header.nanosec // 1000 timestamp = timelib.Timestamp.FromPosixTimeWithMicrosecond( record_header.timestamp, microsecond) record_position = offset message_id = record_header.asl_message_id level = u'{0} ({1})'.format( self.ASL_MESSAGE_PRIORITY[record_header.level], record_header.level) # If the value is -1 (FFFFFFFF), it can be read by everyone. if record_header.read_uid != int(self.ASL_NO_RIGHTS, 16): read_uid = record_header.read_uid else: read_uid = 'ALL' if record_header.read_gid != int(self.ASL_NO_RIGHTS, 16): read_gid = record_header.read_gid else: read_gid = 'ALL' # Parsing the dynamic values (text or pointers to position with text). # The first four are always the host, sender, facility, and message. computer_name = values[0] sender = values[1] facility = values[2] message = values[3] # If the entry has an extra fields, they works as a pairs: # The first is the name of the field and the second the value. extra_information = '' if len(values) > 4: values = values[4:] for index in xrange(0, len(values) // 2): extra_information += (u'[{0}: {1}]'.format( values[index * 2], values[(index * 2) + 1])) # Return the event and the offset for the next entry. return AslEvent(timestamp, record_position, message_id, level, record_header, read_uid, read_gid, computer_name, sender, facility, message, extra_information), record_header.next_offset
decoder=lambda obj, ctx: ''.join(dsb.data_values for dsb in obj.blocks), ) _image_block = construct.Struct( 'image', construct.Value('block_type', lambda ctx: 'image'), construct.Struct( 'image_descriptor', construct.ULInt16('left'), construct.ULInt16('top'), construct.ULInt16('width'), construct.ULInt16('height'), construct.EmbeddedBitStruct( construct.Flag('lct_flag'), construct.Flag('interlace_flag'), construct.Flag('sort_flag'), construct.Padding(2), # reserved construct.macros.BitField('lct_size', 3), ), ), construct.If( lambda ctx: ctx.image_descriptor.lct_flag, construct.Array( lambda ctx: pow(2, ctx.image_descriptor.lct_size + 1), construct.Array(3, construct.ULInt8('lct')), ), ), construct.ULInt8('lzw_min'), _get_data_subblocks('compressed_indices'),
class ASLParser(interface.FileObjectParser): """Parser for ASL log files.""" _INITIAL_FILE_OFFSET = None NAME = u'asl_log' DESCRIPTION = u'Parser for ASL log files.' _ASL_MAGIC = b'ASL DB\x00\x00\x00\x00\x00\x00' # ASL File header. # magic: magic number that identify ASL files. # version: version of the file. # offset: first record in the file. # timestamp: time when the first entry was written. # Contains the number of seconds since January 1, 1970 00:00:00 UTC. # last_offset: last record in the file. _ASL_HEADER_STRUCT = construct.Struct(u'asl_header_struct', construct.String(u'magic', 12), construct.UBInt32(u'version'), construct.UBInt64(u'offset'), construct.UBInt64(u'timestamp'), construct.UBInt32(u'cache_size'), construct.UBInt64(u'last_offset'), construct.Padding(36)) # The record structure is: # [HEAP][STRUCTURE][4xExtraField][2xExtraField]*[PreviousEntry] # Record static structure. # tam_entry: it contains the number of bytes from this file position # until the end of the record, without counts itself. # next_offset: next record. If is equal to 0x00, it is the last record. # asl_message_id: integer that has the numeric identification of the event. # timestamp: the entry creation date and time. # Contains the number of seconds since January 1, 1970 00:00:00 UTC. # nanosecond: nanosecond to add to the timestamp. # level: level of priority. # pid: process identification that ask to save the record. # uid: user identification that has lunched the process. # gid: group identification that has lunched the process. # read_uid: identification id of a user. Only applied if is not -1 (all FF). # Only root and this user can read the entry. # read_gid: the same than read_uid, but for the group. _ASL_RECORD_STRUCT = construct.Struct(u'asl_record_struct', construct.Padding(2), construct.UBInt32(u'tam_entry'), construct.UBInt64(u'next_offset'), construct.UBInt64(u'asl_message_id'), construct.UBInt64(u'timestamp'), construct.UBInt32(u'nanosec'), construct.UBInt16(u'level'), construct.UBInt16(u'flags'), construct.UBInt32(u'pid'), construct.UBInt32(u'uid'), construct.UBInt32(u'gid'), construct.UBInt32(u'read_uid'), construct.UBInt32(u'read_gid'), construct.UBInt64(u'ref_pid')) _ASL_RECORD_STRUCT_SIZE = _ASL_RECORD_STRUCT.sizeof() # 8-byte fields, they can be: # - String: [Nibble = 1000 (8)][Nibble = Length][7 Bytes = String]. # - Integer: integer that has the byte position in the file that points # to an ASL_RECORD_DYN_VALUE struct. If the value of the integer # is equal to 0, it means that it has not data (skip). # If the field is a String, we use this structure to decode each # integer byte in the corresponding character (ASCII Char). _ASL_OCTET_STRING = construct.ExprAdapter( construct.Octet(u'string'), encoder=lambda obj, ctx: ord(obj), decoder=lambda obj, ctx: chr(obj)) # Field string structure. If the first bit is 1, it means that it # is a String (1000) = 8, then the next nibble has the number of # characters. The last 7 bytes are the number of bytes. _ASL_STRING = construct.BitStruct( u'string', construct.Flag(u'type'), construct.Bits(u'filler', 3), construct.If(lambda ctx: ctx.type, construct.Nibble(u'string_length')), construct.If(lambda ctx: ctx.type, construct.Array(7, _ASL_OCTET_STRING))) # 8-byte pointer to a byte position in the file. _ASL_POINTER = construct.UBInt64(u'pointer') # Dynamic data structure pointed by a pointer that contains a String: # [2 bytes padding][4 bytes size of String][String]. _ASL_RECORD_DYN_VALUE = construct.Struct( u'asl_record_dyn_value', construct.Padding(2), construct.UBInt32(u'size'), construct.Bytes(u'value', lambda ctx: ctx.size)) def ParseFileObject(self, parser_mediator, file_object, **kwargs): """Parses an ALS file-like object. Args: parser_mediator: a parser mediator object (instance of ParserMediator). file_object: a file-like object. Raises: UnableToParseFile: when the file cannot be parsed. """ file_object.seek(0, os.SEEK_SET) try: header = self._ASL_HEADER_STRUCT.parse_stream(file_object) except (IOError, construct.FieldError) as exception: raise errors.UnableToParseFile( u'Unable to parse ASL Header with error: {0:s}.'.format( exception)) if header.magic != self._ASL_MAGIC: raise errors.UnableToParseFile( u'Not an ASL Header, unable to parse.') offset = header.offset if not offset: return header_last_offset = header.last_offset previous_offset = offset event_object, offset = self.ReadASLEvent(parser_mediator, file_object, offset) while event_object: # Sanity check, the last read element must be the same as # indicated by the header. if offset == 0 and previous_offset != header_last_offset: parser_mediator.ProduceParseError( u'Unable to parse header. Last element header does not match ' u'header offset.') previous_offset = offset event_object, offset = self.ReadASLEvent(parser_mediator, file_object, offset) def ReadASLEvent(self, parser_mediator, file_object, offset): """Reads an ASL record at a specific offset. Args: parser_mediator: a parser mediator object (instance of ParserMediator). file_object: a file-like object that points to an ASL file. offset: an integer containing the offset of the ASL record. Returns: A tuple of an event object extracted from the ASL record, and the offset to the next ASL record in the file. """ # The heap of the entry is saved to try to avoid seek (performance issue). # It has the real start position of the entry. dynamic_data_offset = file_object.tell() try: dynamic_data = file_object.read(offset - dynamic_data_offset) except IOError as exception: parser_mediator.ProduceParseError( u'unable to read ASL record dynamic data with error: {0:s}'. format(exception)) return None, None if not offset: return None, None try: record_struct = self._ASL_RECORD_STRUCT.parse_stream(file_object) except (IOError, construct.FieldError) as exception: parser_mediator.ProduceParseError( u'unable to parse ASL record with error: {0:s}'.format( exception)) return None, None # Variable tam_fields = is the real length of the dynamic fields. # We have this: [Record_Struct] + [Dynamic_Fields] + [Pointer_Entry_Before] # In Record_Struct we have a field called tam_entry, where it has the number # of bytes until the end of the entry from the position that the field is. # The tam_entry is between the 2th and the 6th byte in the [Record_Struct]. # tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before] # Also, we do not need [Point_Entry_Before] and then we delete the size of # [Point_Entry_Before] that it is 8 bytes (8): # tam_entry = ([Record_Struct]-6)+[Dynamic_Fields]+[Pointer_Entry_Before] # [Dynamic_Fields] = tam_entry - [Record_Struct] + 6 - 8 # [Dynamic_Fields] = tam_entry - [Record_Struct] - 2 tam_fields = record_struct.tam_entry - self._ASL_RECORD_STRUCT_SIZE - 2 # Dynamic part of the entry that contains minimal four fields of 8 bytes # plus 2 x [8 bytes] fields for each extra ASL_Field. # The four first fields are always the Host, Sender, Facility and Message. # After the four first fields, the entry might have extra ASL_Fields. # For each extra ASL_field, it has a pair of 8-byte fields where the first # 8 bytes contains the name of the extra ASL_field and the second 8 bytes # contains the text of the extra field. # All of this 8-byte field can be saved using one of these three different # types: # - Null value ('0000000000000000'): nothing to do. # - String: It is string if first bit = 1 or first nibble = 8 (1000). # Second nibble has the length of string. # The next 7 bytes have the text characters of the string # padding the end with null characters: '0x00'. # Example: [8468 6964 6400 0000] # [8] String, [4] length, value: [68 69 64 64] = hidd. # - Pointer: static position in the file to a special struct # implemented as an ASL_RECORD_DYN_VALUE. # Example: [0000 0000 0000 0077] # It points to the file position 0x077 that has a # ASL_RECORD_DYN_VALUE structure. values = [] while tam_fields > 0: try: field_data = file_object.read(8) except IOError as exception: parser_mediator.ProduceParseError( u'unable to read ASL field with error: {0:s}'.format( exception)) return None, None # Try to read the field data as a string. try: asl_string_struct = self._ASL_STRING.parse(field_data) string_data = b''.join( asl_string_struct.string[0:asl_string_struct. string_length]) values.append(string_data) # Go to parse the next extra field. tam_fields -= 8 continue except ValueError: pass # If the field is not a string it must be a pointer. try: pointer_value = self._ASL_POINTER.parse(field_data) except ValueError as exception: parser_mediator.ProduceParseError( u'unable to parse ASL field with error: {0:s}'.format( exception)) return None, None if not pointer_value: # Next extra field: 8 bytes more. tam_fields -= 8 continue # The next IF ELSE is only for performance issues, avoiding seek. # If the pointer points a lower position than where the actual entry # starts, it means that it points to a previous entry. pos = pointer_value - dynamic_data_offset # Greater or equal 0 means that the data is in the actual entry. if pos >= 0: try: dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse( dynamic_data[pos:]) dyn_value = dyn_value_struct.value.partition(b'\x00')[0] values.append(dyn_value) except (IOError, construct.FieldError) as exception: parser_mediator.ProduceParseError(( u'unable to parse ASL record dynamic value with error: ' u'{0:s}').format(exception)) return None, None else: # Only if it is a pointer that points to the # heap from another entry we use the seek method. main_position = file_object.tell() # If the pointer is in a previous entry. if main_position > pointer_value: file_object.seek(pointer_value - main_position, os.SEEK_CUR) try: dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream( file_object) dyn_value = dyn_value_struct.value.partition( b'\x00')[0] values.append(dyn_value) except (IOError, construct.FieldError): parser_mediator.ProduceParseError(( u'the pointer at {0:d} (0x{0:08x}) points to invalid ' u'information.' ).format(main_position - self._ASL_POINTER.sizeof())) # Come back to the position in the entry. _ = file_object.read(main_position - file_object.tell()) else: _ = file_object.read(pointer_value - main_position) dyn_value_struct = self._ASL_RECORD_DYN_VALUE.parse_stream( file_object) dyn_value = dyn_value_struct.value.partition(b'\x00')[0] values.append(dyn_value) # Come back to the position in the entry. file_object.seek(main_position - file_object.tell(), os.SEEK_CUR) # Next extra field: 8 bytes more. tam_fields -= 8 # Read the last 8 bytes of the record that points to the previous entry. _ = file_object.read(8) # Parsed section, we translate the read data to an appropriate format. micro_seconds, _ = divmod(record_struct.nanosec, 1000) # Parsing the dynamic values (text or pointers to position with text). # The first four are always the host, sender, facility, and message. number_of_values = len(values) if number_of_values < 4: parser_mediator.ProduceParseError( u'less than four values read from an ASL event.') computer_name = u'N/A' sender = u'N/A' facility = u'N/A' message = u'N/A' if number_of_values >= 1: computer_name = values[0].decode(u'utf-8') if number_of_values >= 2: sender = values[1].decode(u'utf-8') if number_of_values >= 3: facility = values[2].decode(u'utf-8') if number_of_values >= 4: message = values[3].decode(u'utf-8') # If the entry has an extra fields, they works as a pairs: # The first is the name of the field and the second the value. extra_information = u'' if number_of_values > 4 and number_of_values % 2 == 0: # Taking all the extra attributes and merging them together, # eg: a = [1, 2, 3, 4] will look like "1: 2, 3: 4". try: extra_values = map(py2to3.UNICODE_TYPE, values[4:]) extra_information = u', '.join( map(u': '.join, zip(extra_values[0::2], extra_values[1::2]))) except UnicodeDecodeError as exception: parser_mediator.ProduceParseError( u'Unable to decode all ASL values in the extra information fields.' ) event_object = ASLEvent(record_struct.timestamp, offset, record_struct.asl_message_id, record_struct.level, record_struct.pid, record_struct.uid, record_struct.gid, record_struct.read_uid, record_struct.read_gid, computer_name, sender, facility, message, extra_information, micro_seconds=micro_seconds) parser_mediator.ProduceEvent(event_object) return (event_object, record_struct.next_offset)
tx_core_name = 'gbe0' test_core_bram = 'qdrBram' test_QDRReg = 'testQDR' test_fineFFT = 'vals_testQDR' fpga = [] snap_debug = 'snap_debug' snap_fengine_debug_coarse_fft = construct.BitStruct( snap_debug, construct.Padding(128 - (4 * 18)), construct.BitField("d0_r", 18), construct.BitField("d0_i", 18), construct.BitField("d1_r", 18), construct.BitField("d1_i", 18)) # the xaui snap block on the f-engine - this is just after packetisation snap_fengine_xaui = construct.BitStruct( "snap_debug", construct.Padding(128 - 1 - 3 - 1 - 1 - 3 - 64), construct.Flag("link_down"), construct.Padding(3), construct.Flag("mrst"), construct.Padding(1), construct.Flag("eof"), construct.Flag("sync"), construct.Flag("hdr_valid"), construct.BitField("data", 64)) def bin2fp(bits, m=8, e=7): if m > 32: raise RuntimeError('Unsupported fixed format: %i.%i' % (m, e)) shift = 32 - m bits = bits << shift m = m + shift e = e + shift return float(numpy.int32(bits)) / (2**e) def bram(bname, odata, samples=1024):
import construct, corr_functions register_fengine_control = construct.BitStruct( 'feng_ctl', construct.Flag('gbe_gpu_rst'), #31 construct.Flag('gbe_sw_rst'), #30 construct.Flag('loopback_mux_rst'), #29 construct.Flag('cnt_rst'), #28 construct.BitField('fft_preshift', 2), #26-27 construct.BitField('gpio_monsel', 3), #23,24,25 construct.Flag('fft_tvg2'), #22 construct.Flag('fft_tvg1'), #21 construct.Flag('gbe_gpu_disable'), #20 construct.Flag('use_qdr_tvg'), #19 construct.Flag('gbe_sw_disable'), #18 construct.Flag('arm_rst'), #17 construct.Flag('sync_rst'), #16 construct.Flag('arm_noise'), #15 construct.Padding(12), #3-14 construct.Flag('lb_err_cnt_rst'), #2 construct.Padding(2)) #0-1
# from . import snap, corr_functions def bin2fp(bits, m = 8, e = 7): if m > 32: raise RuntimeError('Unsupported fixed format: %i.%i' % (m,e)) shift = 32 - m bits = bits << shift m = m + shift e = e + shift return float(numpy.int32(bits)) / (2**e) if construct.version[1] <= 6: # f-engine adc control register_fengine_adc_control = construct.BitStruct('adc_ctrl0', construct.Flag('enable'), # 31 Enable input channel on KAT ADC. construct.Padding(32 - 6 - 1), # 6-30 construct.BitField('atten', 6)) # 0-5 KAT ADC channel attenuation setting. # f-engine status register_fengine_fstatus = construct.BitStruct('fstatus0', construct.BitField('coarse_bits', 5), # 27-31 2^x - the number of points in the coarse FFT. construct.BitField('fine_bits', 5), # 22-26 2^y - the number of points in the fine FFT. construct.BitField('sync_val', 2), # 20-21 On which ADC cycle did the sync happen? construct.Padding(2), # 18-19 construct.Flag('xaui_lnkdn'), # 17 The 10GBE link is down. construct.Flag('xaui_over'), # 16 The 10GBE link has overflows. construct.Padding(9), # 7-15 construct.Flag('clk_err'), # 6 The board frequency is calculated out of bounds. construct.Flag('adc_disabled'), # 5 The ADC has been disabled. construct.Flag('ct_error'), # 4 There is a QDR error from the corner-turner.
class FlacReader: FRAME_HEADER = construct.Struct( 'frame_header', construct.Bits('sync', 14), construct.Bits('reserved', 2), construct.Bits('block_size', 4), construct.Bits('sample_rate', 4), construct.Bits('channel_assignment', 4), construct.Bits('bits_per_sample', 3), construct.Padding(1), construct.IfThenElse( 'total_channels', lambda ctx1: ctx1['channel_assignment'] <= 7, construct.Value('c', lambda ctx2: ctx2['channel_assignment'] + 1), construct.Value('c', lambda ctx3: 2)), UTF8('frame_number'), construct.IfThenElse( 'extended_block_size', lambda ctx1: ctx1['block_size'] == 6, construct.Bits('b', 8), construct.If(lambda ctx2: ctx2['block_size'] == 7, construct.Bits('b', 16))), construct.IfThenElse( 'extended_sample_rate', lambda ctx1: ctx1['sample_rate'] == 12, construct.Bits('s', 8), construct.If(lambda ctx2: ctx2['sample_rate'] in (13, 14), construct.Bits('s', 16))), construct.Bits('crc8', 8)) UNARY = construct.Struct( 'unary', construct.RepeatUntil(lambda obj, ctx: obj == '\x01', construct.Field('bytes', 1)), construct.Value('value', lambda ctx: len(ctx['bytes']) - 1)) SUBFRAME_HEADER = construct.Struct( 'subframe_header', construct.Padding(1), construct.Bits('subframe_type', 6), construct.Flag('has_wasted_bits_per_sample'), construct.IfThenElse('wasted_bits_per_sample', lambda ctx: ctx['has_wasted_bits_per_sample'], PlusOne(Unary('value')), construct.Value('value', lambda ctx2: 0))) GET_BLOCKSIZE_FROM_STREAMINFO = -1 GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER = -2 GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER = -3 BLOCK_SIZE = (GET_BLOCKSIZE_FROM_STREAMINFO, 192, 576, 1152, 2304, 4608, GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER, GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768) GET_SAMPLE_SIZE_FROM_STREAMINFO = -1 SAMPLE_SIZE = (GET_SAMPLE_SIZE_FROM_STREAMINFO, 8, 12, None, 16, 20, 24, None) def FIXED0(subframe, residual, i): subframe.insert(i, residual[i]) def FIXED1(subframe, residual, i): subframe.insert(i, subframe[i - 1] + residual[i]) def FIXED2(subframe, residual, i): subframe.insert(i, ((2 * subframe[i - 1]) - subframe[i - 2] + \ residual[i])) def FIXED3(subframe, residual, i): subframe.insert(i, ((3 * subframe[i - 1]) - (3 * subframe[i - 2]) + \ subframe[i - 3] + residual[i])) def FIXED4(subframe, residual, i): subframe.insert(i, ((4 * subframe[i - 1]) - (6 * subframe[i - 2]) + \ (4 * subframe[i - 3]) - subframe[i - 4] + residual[i])) #iterates over all of the channels, in order def MERGE_INDEPENDENT(channel_list): channel_data = [iter(c) for c in channel_list] while (True): for channel in channel_data: yield channel.next() def MERGE_LEFT(channel_list): channel_left = iter(channel_list[0]) channel_side = iter(channel_list[1]) while (True): left = channel_left.next() side = channel_side.next() yield left yield left - side def MERGE_RIGHT(channel_list): channel_side = iter(channel_list[0]) channel_right = iter(channel_list[1]) while (True): side = channel_side.next() right = channel_right.next() yield side + right yield right def MERGE_MID(channel_list): channel_mid = iter(channel_list[0]) channel_side = iter(channel_list[1]) while (True): mid = channel_mid.next() side = channel_side.next() mid = mid << 1 mid |= (side & 0x1) yield (mid + side) >> 1 yield (mid - side) >> 1 CHANNEL_FUNCTIONS = (MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_INDEPENDENT, MERGE_LEFT, MERGE_RIGHT, MERGE_MID) FIXED_FUNCTIONS = (FIXED0, FIXED1, FIXED2, FIXED3, FIXED4) def __init__(self, flac_stream): self.stream = BufferedStream(flac_stream) self.streaminfo = None self.bitstream = None #ensure the file starts with 'fLaC' self.read_stream_marker() #initialize self.bitstream self.begin_bitstream() #find self.streaminfo in case we need it self.read_metadata_blocks() def close(self): if (self.bitstream != None): self.bitstream.close() else: self.stream.close() def read_stream_marker(self): if (self.stream.read(4) != 'fLaC'): raise FlacStreamException('invalid stream marker') def read_metadata_blocks(self): block = audiotools.FlacAudio.METADATA_BLOCK_HEADER.parse_stream( self.stream) while (block.last_block == 0): if (block.block_type == 0): self.streaminfo = audiotools.FlacAudio.STREAMINFO.parse_stream( self.stream) else: self.stream.seek(block.block_length, 1) block = audiotools.FlacAudio.METADATA_BLOCK_HEADER.parse_stream( self.stream) self.stream.seek(block.block_length, 1) def begin_bitstream(self): import bitstream #self.bitstream = construct.BitStreamReader(self.stream) self.bitstream = bitstream.BitStreamReader(self.stream) def read_frame(self): self.stream.reset_buffer() try: header = FlacReader.FRAME_HEADER.parse_stream(self.bitstream) except construct.core.FieldError: return "" if (header.sync != 0x3FFE): raise FlacStreamException('invalid sync') if (crc8(self.stream.getvalue()[0:-1]) != header.crc8): raise FlacStreamException('crc8 checksum failed') #block_size tells us how many samples we need from each subframe block_size = FlacReader.BLOCK_SIZE[header.block_size] if (block_size == self.GET_BLOCKSIZE_FROM_STREAMINFO): block_size = self.streaminfo.maximum_blocksize elif ((block_size == self.GET_8BIT_BLOCKSIZE_FROM_END_OF_HEADER) or (block_size == self.GET_16BIT_BLOCKSIZE_FROM_END_OF_HEADER)): block_size = header.extended_block_size + 1 #grab subframe data as 32-bit array objects subframe_data = [] for channel_number in xrange(header.total_channels): subframe_data.append( self.read_subframe(header, block_size, channel_number)) crc16sum = crc16(self.stream.getvalue()) #try to byte-align the stream if (len(self.bitstream.buffer) > 0): self.bitstream.read(len(self.bitstream.buffer)) if (crc16sum != construct.Bits('crc16', 16).parse_stream( self.bitstream)): raise FlacStreamException('crc16 checksum failed') #convert our list of subframe data arrays into #a string of sample data if (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == 16): merged_frames = array.array( 'h', FlacReader.CHANNEL_FUNCTIONS[header.channel_assignment]( subframe_data)) if (audiotools.BIG_ENDIAN): merged_frames.byteswap() return merged_frames.tostring() elif (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == 8): merged_frames = array.array( 'b', FlacReader.CHANNEL_FUNCTIONS[header.channel_assignment]( subframe_data)) return merged_frames.tostring() else: if (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == \ self.GET_SAMPLE_SIZE_FROM_STREAMINFO): bits_per_sample = self.streaminfo.bits_per_sample + 1 elif (FlacReader.SAMPLE_SIZE[header.bits_per_sample] == None): raise FlacStreamException('invalid bits per sample') else: bits_per_sample = FlacReader.SAMPLE_SIZE[ header.bits_per_sample] stream = construct.GreedyRepeater( construct.BitStruct( 'bits', construct.Bits('value', bits_per_sample, swapped=True, signed=True))) return stream.build([ construct.Container(value=v) for v in FlacReader.CHANNEL_FUNCTIONS[ header.channel_assignment](subframe_data) ]) def read_subframe(self, frame_header, block_size, channel_number): subframe_header = \ FlacReader.SUBFRAME_HEADER.parse_stream(self.bitstream) #figure out the bits-per-sample of this subframe if ((frame_header.channel_assignment == 8) and (channel_number == 1)): #if channel is stored as left+difference #and this is the difference, add 1 bit bits_per_sample = FlacReader.SAMPLE_SIZE[ frame_header.bits_per_sample] + 1 elif ((frame_header.channel_assignment == 9) and (channel_number == 0)): #if channel is stored as difference+right #and this is the difference, add 1 bit bits_per_sample = FlacReader.SAMPLE_SIZE[ frame_header.bits_per_sample] + 1 elif ((frame_header.channel_assignment == 10) and (channel_number == 1)): #if channel is stored as average+difference #and this is the difference, add 1 bit bits_per_sample = FlacReader.SAMPLE_SIZE[ frame_header.bits_per_sample] + 1 else: #otherwise, use the number from the frame header bits_per_sample = FlacReader.SAMPLE_SIZE[ frame_header.bits_per_sample] if (subframe_header.has_wasted_bits_per_sample): bits_per_sample -= subframe_header.wasted_bits_per_sample if (subframe_header.subframe_type == 0): subframe = self.read_subframe_constant(block_size, bits_per_sample) elif (subframe_header.subframe_type == 1): subframe = self.read_subframe_verbatim(block_size, bits_per_sample) elif ((subframe_header.subframe_type & 0x38) == 0x08): subframe = self.read_subframe_fixed( subframe_header.subframe_type & 0x07, block_size, bits_per_sample) elif ((subframe_header.subframe_type & 0x20) == 0x20): subframe = self.read_subframe_lpc( (subframe_header.subframe_type & 0x1F) + 1, block_size, bits_per_sample) else: raise FlacStreamException('invalid subframe type') if (subframe_header.has_wasted_bits_per_sample): return array.array('i', [ i << subframe_header.wasted_bits_per_sample for i in subframe ]) else: return subframe def read_subframe_constant(self, block_size, bits_per_sample): sample = construct.Bits('b', bits_per_sample).parse_stream(self.bitstream) subframe = array.array('i', [sample] * block_size) return subframe def read_subframe_verbatim(self, block_size, bits_per_sample): return array.array( 'i', construct.StrictRepeater( block_size, construct.Bits("samples", bits_per_sample, signed=True)).parse_stream(self.bitstream)) def read_subframe_fixed(self, order, block_size, bits_per_sample): samples = construct.StrictRepeater( order, construct.Bits("warm_up_samples", bits_per_sample, signed=True)) subframe = array.array('i', samples.parse_stream(self.bitstream)) residual = self.read_residual(block_size, order) fixed_func = self.FIXED_FUNCTIONS[order] for i in xrange(len(subframe), block_size): fixed_func(subframe, residual, i) return subframe def read_subframe_lpc(self, order, block_size, bits_per_sample): samples = construct.StrictRepeater( order, construct.Bits("warm_up_samples", bits_per_sample, signed=True)) subframe = array.array('i', samples.parse_stream(self.bitstream)) lpc_precision = construct.Bits('lpc_precision', 4).parse_stream( self.bitstream) + 1 lpc_shift = construct.Bits('lpc_shift', 5).parse_stream(self.bitstream) coefficients = array.array( 'i', construct.StrictRepeater( order, construct.Bits('coefficients', lpc_precision, signed=True)).parse_stream(self.bitstream)) residual = self.read_residual(block_size, order) for i in xrange(len(subframe), block_size): subframe.insert(i, (sum( [coefficients[j] * subframe[i - j - 1] for j in xrange(0,len(coefficients))]) >> lpc_shift) + \ residual[i]) return subframe def read_residual(self, block_size, predictor_order): rice = array.array('i') #add some dummy rice so that the Rice index matches #that of the rest of the subframe for i in xrange(predictor_order): rice.append(0) coding_method = self.bitstream.read(2) if (coding_method == '\x00\x00'): rice2 = False elif (coding_method == '\x00\x01'): rice2 = True else: raise FlacStreamException('invalid residual coding method') partition_order = construct.Bits('partition_order', 4).parse_stream(self.bitstream) if (partition_order > 0): total_samples = ((block_size / 2**partition_order) - predictor_order) rice.extend(self.read_encoded_rice(total_samples, rice2)) for i in xrange(1, 2**partition_order): total_samples = (block_size / 2**partition_order) rice.extend(self.read_encoded_rice(total_samples, rice2)) else: rice.extend( self.read_encoded_rice(block_size - predictor_order, rice2)) return rice def read_encoded_rice(self, total_samples, rice2=False): bin_to_int = construct.lib.binary.bin_to_int samples = array.array('i') if (not rice2): rice_parameter = construct.Bits('rice_parameter', 4).parse_stream(self.bitstream) else: rice_parameter = construct.Bits('rice_parameter', 5).parse_stream(self.bitstream) if (rice_parameter != 0xF): #a Rice encoded residual for x in xrange(total_samples): #count the number of 0 bits before the next 1 bit #(unary encoding) #to find our most significant bits msb = 0 s = self.bitstream.read(1) while (s != '\x01'): msb += 1 s = self.bitstream.read(1) #grab the proper number of least significant bits lsb = bin_to_int(self.bitstream.read(rice_parameter)) #combine msb and lsb to get the Rice-encoded value value = (msb << rice_parameter) | lsb if ((value & 0x1) == 0x1): #negative samples.append(-(value >> 1) - 1) else: #positive samples.append(value >> 1) else: #unencoded residual bits_per_sample = construct.Bits('escape_code', 5).parse_stream(self.bitstream) sample = construct.Bits("sample", bits_per_sample, signed=True) for x in xrange(total_samples): samples.append(sample.parse_stream(self.bitstream)) return samples
fine_fft_size = 4096 # Frequency bins offset_shift_size = snap_size # For the fine FFT, when the whole thing can't be measured at once. num_offset_shifts = 4 # In case this later needs to be changed, perhaps for a different size fine FFT. # Variables relevant only to the corner turner snaps_per_spectrum = 128 channels_per_snap = 32 words_per_channel = 32 # Bitstruct to control the control register on the ROACH control_reg_bitstruct = construct.BitStruct( 'control_reg', construct.Padding(4), # 28-31 construct.BitField('debug_snap_select', 3), # 25-27 construct.Padding(3), # 22-24 construct.Flag('fine_tvg_en'), # 21 construct.Flag('adc_tvg'), # 20 construct.Flag('fd_fs_tvg'), # 19 construct.Flag('packetiser_tvg'), # 18 construct.Flag('ct_tvg'), # 17 construct.Flag('tvg_en'), # 16 construct.Padding(4), # 12-15 construct.Flag('fancy_en'), # 11 construct.Flag('adc_protect_disable'), # 10 construct.Flag('gbe_enable'), # 09 construct.Flag('gbe_rst'), # 08 construct.Padding(4), # 04-07 construct.Flag('clr_status'), # 03 construct.Flag('arm'), # 02 construct.Flag('man_sync'), # 01 construct.Flag('sys_rst')) # 00