def we_split_function(self, raw_data): """ Sort through the raw data to identify new blocks of data that need processing. This is needed instead of a regex because blocks are identified by position in this binary file. """ form_list = [] raw_data_len = len(raw_data) """ Ok, there is a new issue with parsing these records. The Status messages can have an optional 2 bytes on the end, and since the rest of the data is relatively unformated packed binary records, detecting the presence of that optional 2 bytes can be difficult. The only pattern we have to detect is the STATUS_START field ( 4 bytes FF FF FF F[A-F] ). So, we peel this appart be parsing backwards, using the end-of-record as an additional anchor point. """ # '-1' to remove the '\x03' end-of-record marker parse_end_point = raw_data_len - 1 while parse_end_point > 0: # look for a status message at postulated message header position header_start = STATUS_BYTES_AUGMENTED # look for an augmented status if STATUS_START_MATCHER.match(raw_data[parse_end_point-STATUS_BYTES_AUGMENTED:parse_end_point]): # A hit for the status message at the augmented offset # NOTE, we don't need the status messages, so we drop them on the floor here # and only deliver a stream of samples to build_parse_values parse_end_point = parse_end_point-STATUS_BYTES_AUGMENTED # check if this is a unaugmented status elif STATUS_START_MATCHER.match(raw_data[parse_end_point-STATUS_BYTES:parse_end_point]): # A hit for the status message at the unaugmented offset # NOTE: same as above parse_end_point = parse_end_point-STATUS_BYTES else: # assume if not a stat that hit above, we have a sample. If we missparse, we # will end up with extra bytes when we finish, and sample_except at that point. form_list.append((parse_end_point-E_GLOBAL_SAMPLE_BYTES, parse_end_point)) parse_end_point = parse_end_point-E_GLOBAL_SAMPLE_BYTES # if the remaining bytes are less than the data sample bytes, all we might have left is a status sample, if we don't we're done if parse_end_point != 0 and parse_end_point < STATUS_BYTES and parse_end_point < E_GLOBAL_SAMPLE_BYTES and parse_end_point < STATUS_BYTES_AUGMENTED: self._exception_callback(UnexpectedDataException("Error sieving WE data, inferred sample/status alignment incorrect")) return_list = [] return return_list # since we parsed this backwards, we need to reverse to list to deliver the data in the correct order return_list = form_list[::-1] log.debug("returning we sieve list %s", return_list) return return_list
def sieve_function(self, raw_data): """ This method sorts through the raw data to identify new blocks of data that need processing. This is needed instead of a regex because blocks are identified by position in this binary file. """ form_list = [] raw_data_len = len(raw_data) # Starting from the end of the buffer and working backwards parse_end_point = raw_data_len # We are going to go through the file data in reverse order since we have a # variable length status indicator field. # While we do not hit the beginning of the file contents, continue while parse_end_point > 0: # Create the different start indices for the three different scenarios raw_data_start_index_augmented = parse_end_point - STATUS_BYTES_AUGMENTED raw_data_start_index_normal = parse_end_point - STATUS_BYTES global_recovered_eng_rec_index = parse_end_point - WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES # Check for an an augmented status first if raw_data_start_index_augmented >= 0 and \ STATUS_START_MATCHER.match(raw_data[raw_data_start_index_augmented:parse_end_point]): log.trace("Found OffloadProfileData with decimation factor") parse_end_point = raw_data_start_index_augmented # Check for a normal status elif raw_data_start_index_normal >= 0 and \ STATUS_START_MATCHER.match(raw_data[raw_data_start_index_normal:parse_end_point]): log.trace("Found OffloadProfileData without decimation factor") parse_end_point = raw_data_start_index_normal # If neither, we are dealing with a global wfp_sio e recovered engineering data record, # so we will save the start and end points elif global_recovered_eng_rec_index >= 0: log.trace("Found OffloadEngineeringData") form_list.append( (global_recovered_eng_rec_index, parse_end_point)) parse_end_point = global_recovered_eng_rec_index # We must not have a good file, log some debug info for now else: log.debug("raw_data_start_index_augmented %d", raw_data_start_index_augmented) log.debug("raw_data_start_index_normal %d", raw_data_start_index_normal) log.debug("global_recovered_eng_rec_index %d", global_recovered_eng_rec_index) log.debug("bad file or bad position?") raise SampleException( "File size is invalid or improper positioning") return_list = form_list[::-1] return return_list
def we_split_function(self, raw_data): """ Sort through the raw data to identify new blocks of data that need processing. """ form_list = [] """ The Status messages can have an optional 2 bytes on the end, and since the rest of the data consists of relatively unformated packed binary records, detecting the presence of that optional 2 bytes can be difficult. The only pattern we have to detect is the STATUS_START field ( 4 bytes FF FF FF F[A-F]). We peel this appart by parsing backwards, using the end-of-record as an additional anchor point. """ parse_end_point = len(raw_data) while parse_end_point > 0: # look for a status message at postulated message header position header_start = STATUS_BYTES_AUGMENTED # look for an augmented status if STATUS_START_MATCHER.match( raw_data[parse_end_point - STATUS_BYTES_AUGMENTED:parse_end_point]): # A hit for the status message at the augmented offset # NOTE, we don't need the status messages and only deliver a stream of # samples to build_parsed_values parse_end_point = parse_end_point - STATUS_BYTES_AUGMENTED # check if this is an unaugmented status elif STATUS_START_MATCHER.match( raw_data[parse_end_point - STATUS_BYTES:parse_end_point]): # A hit for the status message at the unaugmented offset # NOTE: same as above parse_end_point = parse_end_point - STATUS_BYTES else: # assume if not a stat that hit above, we have a sample. Mis-parsing will result # in extra bytes at the end and a sample exception. form_list.append( (parse_end_point - E_GLOBAL_SAMPLE_BYTES, parse_end_point)) parse_end_point = parse_end_point - E_GLOBAL_SAMPLE_BYTES # if the remaining bytes are less than data sample bytes, all we might have left is a status sample if parse_end_point != 0 and parse_end_point < STATUS_BYTES \ and parse_end_point < E_GLOBAL_SAMPLE_BYTES \ and parse_end_point < STATUS_BYTES_AUGMENTED: self._exception_callback( UnexpectedDataException( "Error sieving WE data, inferred sample/status alignment incorrect" )) return_list = [] return return_list # Because we parsed this backwards, we need to reverse the list to deliver the data in the correct order return_list = form_list[::-1] log.debug("returning we sieve/split list %s", return_list) return return_list
def we_split_function(self, raw_data): """ Sort through the raw data to identify new blocks of data that need processing. """ form_list = [] """ The Status messages can have an optional 2 bytes on the end, and since the rest of the data consists of relatively unformated packed binary records, detecting the presence of that optional 2 bytes can be difficult. The only pattern we have to detect is the STATUS_START field ( 4 bytes FF FF FF F[A-F]). We peel this appart by parsing backwards, using the end-of-record as an additional anchor point. """ parse_end_point = len(raw_data) while parse_end_point > 0: # look for a status message at postulated message header position header_start = STATUS_BYTES_AUGMENTED # look for an augmented status if STATUS_START_MATCHER.match(raw_data[parse_end_point - STATUS_BYTES_AUGMENTED : parse_end_point]): # A hit for the status message at the augmented offset # NOTE, we don't need the status messages and only deliver a stream of # samples to build_parsed_values parse_end_point = parse_end_point - STATUS_BYTES_AUGMENTED # check if this is an unaugmented status elif STATUS_START_MATCHER.match(raw_data[parse_end_point - STATUS_BYTES : parse_end_point]): # A hit for the status message at the unaugmented offset # NOTE: same as above parse_end_point = parse_end_point - STATUS_BYTES else: # assume if not a stat that hit above, we have a sample. Mis-parsing will result # in extra bytes at the end and a sample exception. form_list.append((parse_end_point - E_GLOBAL_SAMPLE_BYTES, parse_end_point)) parse_end_point = parse_end_point - E_GLOBAL_SAMPLE_BYTES # if the remaining bytes are less than data sample bytes, all we might have left is a status sample if ( parse_end_point != 0 and parse_end_point < STATUS_BYTES and parse_end_point < E_GLOBAL_SAMPLE_BYTES and parse_end_point < STATUS_BYTES_AUGMENTED ): self._exception_callback( UnexpectedDataException("Error sieving WE data, inferred sample/status alignment incorrect") ) return_list = [] return return_list # Because we parsed this backwards, we need to reverse the list to deliver the data in the correct order return_list = form_list[::-1] log.debug("returning we sieve/split list %s", return_list) return return_list
def sieve_function(self, raw_data): """ This method sorts through the raw data to identify new blocks of data that need processing. This is needed instead of a regex because blocks are identified by position in this binary file. """ form_list = [] raw_data_len = len(raw_data) # Starting from the end of the buffer and working backwards parse_end_point = raw_data_len # We are going to go through the file data in reverse order since we have a # variable length status indicator field. # While we do not hit the beginning of the file contents, continue while parse_end_point > 0: # Create the different start indices for the three different scenarios raw_data_start_index_augmented = parse_end_point-STATUS_BYTES_AUGMENTED raw_data_start_index_normal = parse_end_point-STATUS_BYTES global_recovered_eng_rec_index = parse_end_point-WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES # Check for an an augmented status first if raw_data_start_index_augmented >= 0 and \ STATUS_START_MATCHER.match(raw_data[raw_data_start_index_augmented:parse_end_point]): log.trace("Found OffloadProfileData with decimation factor") parse_end_point = raw_data_start_index_augmented # Check for a normal status elif raw_data_start_index_normal >= 0 and \ STATUS_START_MATCHER.match(raw_data[raw_data_start_index_normal:parse_end_point]): log.trace("Found OffloadProfileData without decimation factor") parse_end_point = raw_data_start_index_normal # If neither, we are dealing with a global wfp_sio e recovered engineering data record, # so we will save the start and end points elif global_recovered_eng_rec_index >= 0: log.trace("Found OffloadEngineeringData") form_list.append((global_recovered_eng_rec_index, parse_end_point)) parse_end_point = global_recovered_eng_rec_index # We must not have a good file, log some debug info for now else: log.debug("raw_data_start_index_augmented %d", raw_data_start_index_augmented) log.debug("raw_data_start_index_normal %d", raw_data_start_index_normal) log.debug("global_recovered_eng_rec_index %d", global_recovered_eng_rec_index) log.debug("bad file or bad position?") raise SampleException("File size is invalid or improper positioning") return_list = form_list[::-1] return return_list
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): # Parse/match the SIO header sio_header_match = SIO_HEADER_MATCHER.match(chunk) end_of_header = sio_header_match.end(0) sample_count = 0 if sio_header_match.group(1) == 'WE': log.trace('read_state: %s', self._read_state) # Parse/match the E file header e_header_match = E_HEADER_MATCHER.search(chunk[end_of_header:end_of_header+HEADER_BYTES]) if e_header_match: payload = chunk[end_of_header+HEADER_BYTES:-1] # '-1' to remove the '\x03' end-of-record marker data_split = self.we_split_function(payload) if data_split: for ii in range(0,len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) self._timestamp = ntplib.system_to_ntp_time(float(fields[0])) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample(FlordLWfpParserDataParticle, None, e_record, self._timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: self._exception_callback(UnexpectedDataException("Found unexpected data.")) else: # no e header match self._exception_callback(UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def _process_we_record(self, payload): indices_list = self.we_split_function(payload) for indices in indices_list: e_record = payload[indices[0] : indices[1]] if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack(">I", e_record[0:4]) self._timestamp = ntplib.system_to_ntp_time(float(fields[0])) sample = self._extract_sample(FlordLWfpSioDataParticle, None, e_record, self._timestamp) if sample: # create particle self._result_particles.append(sample)
def _process_we_record(self, payload): indices_list = self.we_split_function(payload) for indices in indices_list: e_record = payload[indices[0]:indices[1]] if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) self._timestamp = ntplib.system_to_ntp_time(float(fields[0])) sample = self._extract_sample(FlordLWfpSioDataParticle, None, e_record, self._timestamp) if sample: # create particle self._result_particles.append(sample)
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): sio_header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) == 'WE': log.trace("********************************matched chunk header %s", chunk[0:SIO_HEADER_BYTES]) # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search(chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1]) if e_header_match: log.debug('******************************* HEADER MATCH WAS:') log.debug('%s', ":".join("{:02x}".format(ord(c)) for c in chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1])) payload = chunk[SIO_HEADER_BYTES+HEADER_BYTES+1:] data_split = self.we_split_function(payload) if data_split: log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0,len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestampS = float(fields[0]) timestamp = ntplib.system_to_ntp_time(timestampS) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample(DostaLnWfpSioMuleParserDataParticle, None, e_record, timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: # no e header match log.warn("*****************************************************BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback(UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): # Parse/match the SIO header sio_header_match = SIO_HEADER_MATCHER.match(chunk) end_of_header = sio_header_match.end(0) sample_count = 0 if sio_header_match.group(1) == 'WE': log.trace('read_state: %s', self._read_state) # Parse/match the E file header e_header_match = E_HEADER_MATCHER.search( chunk[end_of_header:end_of_header + HEADER_BYTES]) if e_header_match: payload = chunk[ end_of_header + HEADER_BYTES: -1] # '-1' to remove the '\x03' end-of-record marker data_split = self.we_split_function(payload) if data_split: for ii in range(0, len(data_split)): e_record = payload[ data_split[ii][0]:data_split[ii][1]] if not STATUS_START_MATCHER.match( e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) self._timestamp = ntplib.system_to_ntp_time( float(fields[0])) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample( FlordLWfpSioMuleParserDataParticle, None, e_record, self._timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: self._exception_callback( UnexpectedDataException( "Found unexpected data.")) else: # no e header match self._exception_callback( UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def _process_engineering_data(self, profile_eng_data): """ This method processes profile engineering data attempting to create the following particle types along the way: WfpEngWfpSioParserDataStatusParticle WfpEngWfpSioParserDataEngineeringParticle """ # Start from the end of the chunk and working backwards parse_end_point = len(profile_eng_data) # We are going to go through the file data in reverse order since we have a # variable length sample record that could have a decimation factor. # While we do not hit the beginning of the file contents, continue while parse_end_point > 0: # Create the different start indices for the three different scenarios start_index_augmented = parse_end_point-STATUS_BYTES_AUGMENTED start_index_normal = parse_end_point-STATUS_BYTES global_recovered_eng_rec_index = parse_end_point-WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES # Check for an an augmented status first if start_index_augmented >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_augmented:parse_end_point]): log.trace("Found OffloadProfileData with decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_augmented+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioParserDataStatusParticle, None, self._current_controller_timestamp + profile_eng_data[start_index_augmented:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = start_index_augmented # Check for a normal status elif start_index_normal >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_normal:parse_end_point]): log.trace("Found OffloadProfileData without decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_normal+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioParserDataStatusParticle, None, self._current_controller_timestamp + profile_eng_data[start_index_normal:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) parse_end_point = start_index_normal # If neither, we are dealing with a global wfp_sio e recovered engineering data record, # so we will save the start and end points elif global_recovered_eng_rec_index >= 0: log.trace("Found OffloadEngineeringData") fields_prof = struct.unpack_from('>I', profile_eng_data[global_recovered_eng_rec_index:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioParserDataEngineeringParticle, None, profile_eng_data[ global_recovered_eng_rec_index:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = global_recovered_eng_rec_index # We must not have a good file, log some debug info for now else: log.debug("start_index_augmented %d", start_index_augmented) log.debug("start_index_normal %d", start_index_normal) log.debug("global_recovered_eng_rec_index %d", global_recovered_eng_rec_index) self._exception_callback(SampleException("Data invalid")) if sample: log.trace("Sample found: %s", sample.generate()) # create particle self._result_particles.append(sample)
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: sio_header_match = SIO_HEADER_MATCHER.match(chunk) log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) != 'WE': log.warn(" chunk did not match header WE %s", chunk[0:SIO_HEADER_BYTES]) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search( chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES + HEADER_BYTES + 1]) if not e_header_match: # no e header match log.warn("*BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback( UnexpectedDataException("Found unexpected data.")) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop payload = chunk[SIO_HEADER_BYTES + HEADER_BYTES + 1:] data_split = self.we_split_function(payload) log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0, len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] log.debug('Extracted E Record to store in particle %s', hexlify(e_record)) # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestamp_s = float(fields[0]) timestamp = ntplib.system_to_ntp_time(timestamp_s) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: # create particle log.debug('Particle created with raw data %s', hexlify(e_record)) log.debug('Particle timestamp = %f', timestamp) sample = self._extract_sample( DostaLnWfpSioDataParticle, None, e_record, timestamp) result_particles.append(sample) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def we_split_function(self, raw_data): """ Sort through the raw data to identify new blocks of data that need processing. This is needed instead of a regex because blocks are identified by position in this binary file. """ form_list = [] raw_data_len = len(raw_data) """ The Status messages can have an optional 2 bytes on the end, and since the rest of the data is relatively un-formatted packed binary records, detecting the presence of that optional 2 bytes can be difficult. The only pattern we have to detect is the STATUS_START field ( 4 bytes FF FF FF F[A-F] ). Peel this apart by parsing backwards, using the end-of-record as an additional anchor point. """ # '-1' to remove the '\x03' end-of-record marker parse_end_point = raw_data_len - 1 while parse_end_point > 0: # look for a status message at postulated message header position # look for an augmented status if STATUS_START_MATCHER.match( raw_data[parse_end_point - STATUS_BYTES_AUGMENTED:parse_end_point]): # A hit for the status message at the augmented offset # NOTE, we don't need the status messages, so we drop them on the floor here # and only deliver a stream of samples to build_parse_values parse_end_point = parse_end_point - STATUS_BYTES_AUGMENTED # check if this is a un-augmented status elif STATUS_START_MATCHER.match( raw_data[parse_end_point - STATUS_BYTES:parse_end_point]): # A hit for the status message at the un-augmented offset # NOTE: same as above parse_end_point = parse_end_point - STATUS_BYTES else: # assume if not a stat that hit above, we have a sample. If we mis-parse, we # will end up with extra bytes when we finish, and sample_except at that point. form_list.append( (parse_end_point - E_GLOBAL_SAMPLE_BYTES, parse_end_point)) parse_end_point = parse_end_point - E_GLOBAL_SAMPLE_BYTES # if the remaining bytes are less than the data sample bytes, # all we might have left is a status sample, if we don't we're done if parse_end_point != 0 and \ parse_end_point < STATUS_BYTES and \ parse_end_point < E_GLOBAL_SAMPLE_BYTES and \ parse_end_point < STATUS_BYTES_AUGMENTED: self._exception_callback( UnexpectedDataException( "Error sieving WE data, inferred sample/status alignment incorrect" )) return_list = [] return return_list # since we parsed this backwards, we need to reverse to list to deliver the data in the correct order return_list = form_list[::-1] log.debug("returning we sieve list %s", return_list) return return_list
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 # Check to see if we are dealing with a wfp_eng SIO chunk if header_match.group(1) == 'WE': data_match = DATA_MATCHER.match(chunk[len(header_match.group(0)):]) controller_timestamp = header_match.group(3) if data_match: sensor_profile_start_time_data = data_match.group(2) if sensor_profile_start_time_data: # Need to unpack the sensor profile start time timestamp fields_prof = struct.unpack_from('>I', sensor_profile_start_time_data[4:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStartTimeParticle, None, controller_timestamp + sensor_profile_start_time_data, float(ntplib.system_to_ntp_time(timestamp))) if sample: log.debug("Sample found: %s", sample) # create particle result_particles.append(sample) sample_count += 1 profile_eng_data = data_match.group(3) # Start from the end of the chunk and working backwards parse_end_point = len(profile_eng_data) # We are going to go through the file data in reverse order since we have a # variable length sample record that could have a decimation factor. # While we do not hit the beginning of the file contents, continue while parse_end_point > 0: # Create the different start indices for the three different scenarios start_index_augmented = parse_end_point-STATUS_BYTES_AUGMENTED start_index_normal = parse_end_point-STATUS_BYTES global_recovered_eng_rec_index = parse_end_point-WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES # Check for an an augmented status first if start_index_augmented >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_augmented:parse_end_point]): log.debug("Found OffloadProfileData with decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_augmented+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStatusParticle, None, controller_timestamp + profile_eng_data[start_index_augmented:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = start_index_augmented # Check for a normal status elif start_index_normal >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_normal:parse_end_point]): log.debug("Found OffloadProfileData without decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_normal+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStatusParticle, None, controller_timestamp + profile_eng_data[start_index_normal:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) parse_end_point = start_index_normal # If neither, we are dealing with a global wfp e recovered engineering data record, # so we will save the start and end points elif global_recovered_eng_rec_index >= 0: log.debug("Found OffloadEngineeringData") fields_prof = struct.unpack_from('>I', profile_eng_data[global_recovered_eng_rec_index:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataEngineeringParticle, None, profile_eng_data[ global_recovered_eng_rec_index:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = global_recovered_eng_rec_index # We must not have a good file, log some debug info for now else: log.debug("start_index_augmented %d", start_index_augmented) log.debug("start_index_normal %d", start_index_normal) log.debug("global_recovered_eng_rec_index %d", global_recovered_eng_rec_index) self._exception_callback(SampleException("Data invalid")) if sample: log.debug("Sample found: %s", sample) # create particle result_particles.append(sample) sample_count += 1 else: self._exception_callback(SampleException("Unexpected data")) # Important: We must set the sample count for every chunk self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: sio_header_match = SIO_HEADER_MATCHER.match(chunk) log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) != 'WE': log.warn(" chunk did not match header WE %s", chunk[0:SIO_HEADER_BYTES]) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search( chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1]) if not e_header_match: # no e header match log.warn("*BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback(UnexpectedDataException("Found unexpected data.")) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop payload = chunk[SIO_HEADER_BYTES+HEADER_BYTES+1:] data_split = self.we_split_function(payload) log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0, len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] log.debug('Extracted E Record to store in particle %s', hexlify(e_record)) # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestamp_s = float(fields[0]) timestamp = ntplib.system_to_ntp_time(timestamp_s) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: # create particle log.debug('Particle created with raw data %s', hexlify(e_record)) log.debug('Particle timestamp = %f', timestamp) sample = self._extract_sample(DostaLnWfpSioDataParticle, None, e_record, timestamp) result_particles.append(sample) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): sio_header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) == 'WE': log.trace( "********************************matched chunk header %s", chunk[0:SIO_HEADER_BYTES]) # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search( chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES + HEADER_BYTES + 1]) if e_header_match: log.debug( '******************************* HEADER MATCH WAS:') log.debug( '%s', ":".join( "{:02x}".format(ord(c)) for c in chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES + HEADER_BYTES + 1])) payload = chunk[SIO_HEADER_BYTES + HEADER_BYTES + 1:] data_split = self.we_split_function(payload) if data_split: log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0, len(data_split)): e_record = payload[ data_split[ii][0]:data_split[ii][1]] # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match( e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestampS = float(fields[0]) timestamp = ntplib.system_to_ntp_time( timestampS) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample( DostaLnWfpSioMuleParserDataParticle, None, e_record, timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: # no e header match log.warn( "*****************************************************BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback( UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles