def parse_chunks(self): """ Parse chunks for the Recovered CO and CT parser. Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) header_timestamp = header_match.group(SIO_HEADER_GROUP_TIMESTAMP) # # Start processing at the end of the header. # chunk_idx = header_match.end(0) if header_match.group(SIO_HEADER_GROUP_ID) == ID_OFFSET: (particles, had_error) = parse_co_data(CtdmoGhqrSioRecoveredOffsetDataParticle, chunk[chunk_idx:-1], header_timestamp, self._extract_sample) if had_error[0]: log.error('unknown data found in CO chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), had_error[1]) self._exception_callback(SampleException( 'unknown data found in CO chunk at %d, leaving out the rest' % had_error[1])) result_particles.extend(particles) if header_match.group(SIO_HEADER_GROUP_ID) == ID_INSTRUMENT: header_str = header_match.group(0) inductive_id = header_str[8:10] (particles, had_error) = parse_ct_data(CtdmoGhqrRecoveredHostInstrumentDataParticle, chunk[chunk_idx:-1], header_timestamp, self._extract_sample, inductive_id) if had_error[0]: log.error('unknown data found in CT chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), had_error[1]) self._exception_callback(SampleException( 'unknown data found in CT chunk at %d, leaving out the rest' % had_error[1])) result_particles.extend(particles) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', header_match.group(0)[1:32]) if header_match.group(1) == 'DO': data_match = DATA_MATCHER.search(chunk) if data_match: log.debug('Found data match in chunk %s', chunk[1:32]) if not self._read_state.get(StateKey.METADATA_SENT): # create the metadata particle # prepend the timestamp from sio mule header to the dosta raw data, # which is stored in header_match.group(3) metadata_sample = self._extract_sample( DostadMetadataDataParticle, None, header_match.group(3) + data_match.group(0), None) if metadata_sample: result_particles.append(metadata_sample) sample_count += 1 self._read_state[StateKey.METADATA_SENT] = True # create the dosta data particle # prepend the timestamp from sio mule header to the dosta raw data, # which is stored in header_match.group(3) sample = self._extract_sample( DostadParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 self._chunk_sample_count.append(sample_count) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): # Parse/match the SIO header sio_header_match = SIO_HEADER_MATCHER.match(chunk) end_of_header = sio_header_match.end(0) sample_count = 0 if sio_header_match.group(1) == 'WE': log.trace('read_state: %s', self._read_state) # Parse/match the E file header e_header_match = E_HEADER_MATCHER.search(chunk[end_of_header:end_of_header+HEADER_BYTES]) if e_header_match: payload = chunk[end_of_header+HEADER_BYTES:-1] # '-1' to remove the '\x03' end-of-record marker data_split = self.we_split_function(payload) if data_split: for ii in range(0,len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) self._timestamp = ntplib.system_to_ntp_time(float(fields[0])) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample(FlordLWfpParserDataParticle, None, e_record, self._timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: self._exception_callback(UnexpectedDataException("Found unexpected data.")) else: # no e header match self._exception_callback(UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def parse_chunks(self): """ Parse chunks for the Telemetered parser. Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match: header_timestamp = header_match.group(SIO_HEADER_GROUP_TIMESTAMP) # start looping at the end of the header chunk_idx = header_match.end(0) if header_match.group(SIO_HEADER_GROUP_ID) == ID_INSTRUMENT: # # Parse the CT record, up to but not including the end of SIO block. # particles = self.parse_ct_record(chunk[chunk_idx:-1], header_timestamp) result_particles.extend(particles) elif header_match.group(SIO_HEADER_GROUP_ID) == ID_OFFSET: (particles, had_error) = parse_co_data(CtdmoGhqrSioTelemeteredOffsetDataParticle, chunk[chunk_idx:-1], header_timestamp, self._extract_sample) if had_error[0]: log.error('unknown data found in CO chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), had_error[1]) self._exception_callback(SampleException( 'unknown data found in CO chunk at %d, leaving out the rest' % had_error[1])) result_particles.extend(particles) else: message = 'Unexpected Sio Header ID %s' % header_match.group(SIO_HEADER_GROUP_ID) log.warn(message) self._exception_callback(UnexpectedDataException(message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) while chunk is not None: # # Verify that the Instrument ID is the one that we want. # header = SIO_HEADER_MATCHER.match(chunk) if header.group(SIO_HEADER_GROUP_ID) == ID_VEL3D_L_WFP_SIO_MULE: # # Extract the POSIX timestamp from the SIO Header. # sio_timestamp = int(header.group(SIO_HEADER_GROUP_TIMESTAMP), 16) # # Process the remaining Vel3d data, starting from the end of the # SIO Header, but not including the trailing 0x03. # fields = self.parse_vel3d_data(PARTICLE_TYPE_SIO_INSTRUMENT, PARTICLE_TYPE_SIO_METADATA, chunk[header.end(0) : -1], time_stamp=sio_timestamp) # # Generate the particles for this SIO block. # Add them to the return list of particles. # (samples, particles) = self.generate_samples(fields) for x in range(0, samples): result_particles.append(particles[x]) # # Not our instrument, but still must indicate that no samples were found. # else: samples = 0 # keep track of how many samples were found in this chunk self._chunk_sample_count.append(samples) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) # if there is any non data handle it self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match.group(SIO_HEADER_GROUP_ID) == 'CS': data_match = ENG_MATCHER.match(chunk) if data_match: # put timestamp from hex string to float: posix_time = int(header_match.group(SIO_HEADER_GROUP_TIMESTAMP), 16) log.debug('utc timestamp %s', datetime.utcfromtimestamp(posix_time)) timestamp = ntplib.system_to_ntp_time(float(posix_time)) # particle-ize the data block received, return the record sample = self._extract_sample(self._particle_class, None, data_match, internal_timestamp=timestamp) if sample: # create particle result_particles.append(sample) else: log.warn('CS data does not match REGEX') self._exception_callback(SampleException('CS data does not match REGEX')) # 'PS' IDs will also be in this file but are specifically ignored elif header_match.group(SIO_HEADER_GROUP_ID) != 'PS': message = 'Unexpected Sio Header ID %s' % header_match.group(SIO_HEADER_GROUP_ID) log.warn(message) self._exception_callback(UnexpectedDataException(message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) # if there is any non data handle it self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match.group(1) == "DO": data_match = DATA_MATCHER.search(chunk) if data_match: log.debug("Found data match in chunk %s", chunk[1:32]) if not self.metadata_sent: # create the metadata particle # prepend the timestamp from sio mule header to the dosta raw data, # which is stored in header_match.group(3) metadata_sample = self._extract_sample( self._metadata_particle_class, None, (header_match.group(3), data_match), None ) if metadata_sample: result_particles.append(metadata_sample) self.metadata_sent = True # create the dosta data particle # prepend the timestamp from sio mule header to the dosta raw data , # which is stored in header_match.group(3) sample = self._extract_sample( self._data_particle_class, None, (header_match.group(3), data_match), None ) if sample: # create particle result_particles.append(sample) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse chunks for the Telemetered parser. Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) header_timestamp = header_match.group(SIO_HEADER_GROUP_TIMESTAMP) # start looping at the end of the header chunk_idx = header_match.end(0) samples = 0 if header_match.group(SIO_HEADER_GROUP_ID) == ID_INSTRUMENT: # # Parse the CT record, up to but not including the end of SIO block. # (samples, particles) = self.parse_ct_record(chunk[chunk_idx : -1], header_timestamp) if samples > 0: for x in range(0, samples): result_particles.append(particles[x]) elif header_match.group(SIO_HEADER_GROUP_ID) == ID_OFFSET: (samples, particles) = self.parse_co_data( CtdmoTelemeteredOffsetDataParticle, chunk[chunk_idx : -1], header_timestamp) if samples > 0: for x in range(0, samples): result_particles.append(particles[x]) # keep track of how many samples were found in this chunk self._chunk_sample_count.append(samples) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: # Parse/match the SIO header sio_header_match = SIO_HEADER_MATCHER.match(chunk) end_of_header = sio_header_match.end(0) if sio_header_match.group(1) == 'WE': # Parse/match the E file header e_header_match = E_HEADER_MATCHER.search( chunk[end_of_header:end_of_header+HEADER_BYTES]) if e_header_match: # '-1' to remove the '\x03' end-of-record marker payload = chunk[end_of_header+HEADER_BYTES:-1] self._process_we_record(payload) else: message = "Found unexpected data." log.warn(message) self._exception_callback(UnexpectedDataException( message)) else: # no e header match message = "Found unexpected data." log.warn(message) self._exception_callback(UnexpectedDataException(message)) (timestamp, chunk) = self._chunker.get_next_data() return self._result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match.group(1) == 'FL': data_match = DATA_MATCHER.search(chunk) if data_match: log.debug('Found data match in chunk %s', chunk[1:32]) # particle-ize the data block received, return the record # prepend the timestamp from sio mule header to the flort raw data, # which is stored in header_match.group(3) sample = self._extract_sample(self._particle_class, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) else: # We found a line in the file that was unexpected. Since we are continuing, # just log a warning. warn_str = "Unexpected data in beginning of header: " log.warn(warn_str + "%s", header_match.group(1)) message = warn_str + header_match.group(1) self._exception_callback(UnexpectedDataException(message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse chunks for the Recovered CO parser. Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) header_timestamp = header_match.group(SIO_HEADER_GROUP_TIMESTAMP) # # Start processing at the end of the header. # chunk_idx = header_match.end(0) if header_match.group(SIO_HEADER_GROUP_ID) == ID_OFFSET: (samples, particles) = self.parse_co_data( CtdmoRecoveredOffsetDataParticle, chunk[chunk_idx : -1], header_timestamp) for x in range(0, samples): result_particles.append(particles[x]) else: samples = 0 # keep track of how many samples were found in this chunk self._chunk_sample_count.append(samples) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', header_match.group(0)[1:32]) if header_match.group(1) == 'CS': log.debug('\n\nCS Header detected::: %s\n\n', header_match.group(0)[1:32]) data_match = ENG_MATCHER.match(chunk) if data_match: # put timestamp from hex string to float: posix_time = int(header_match.group(3), 16) log.debug('utc timestamp %s', datetime.utcfromtimestamp(posix_time)) self._timestamp = ntplib.system_to_ntp_time(float(posix_time)) # particle-ize the data block received, return the record sample = self._extract_sample(self._particle_class, ENG_MATCHER, chunk, self._timestamp) if sample: # create particle result_particles.append(sample) sample_count +=1 else: log.warn('CS data does not match REGEX') self._exception_callback(SampleException('CS data does not match REGEX')) self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() sample_count = 0 while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', header_match.group(0)[1:32]) if header_match.group(1) == 'DO': data_match = DATA_MATCHER.search(chunk) if data_match: log.debug('Found data match in chunk %s', chunk[1:32]) # particle-ize the data block received, return the record sample = self._extract_sample(DostadParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 self._chunk_sample_count.append(sample_count) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) # Check to see if we are dealing with a wfp_eng SIO chunk if header_match.group(1) == 'WE': self._current_controller_timestamp = header_match.group(3) self._process_wfp_eng_chunk(chunk[len(header_match.group(0)):]) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return self._result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 if header_match.group(1) == 'PH': # start after the sio header index = header_match.end(0) last_index = index chunk_len = len(chunk) while index < chunk_len: data_match = DATA_MATCHER.match(chunk[index:]) control_match = CONTROL_MATCHER.match(chunk[index:]) # check for any valid match and make sure no extra data was found between valid matches if data_match or control_match or chunk[index] == SIO_END: # if the indices don't match we have data that doesn't match # exclude the expected possible ph end bytes if last_index != index and chunk[ last_index:index] != PH_EXTRA_END: # we found bad data, send a sample exception but keep processing the file log.warning( "unknown data found in chunk %s from %d to %d", chunk[1:32], last_index, index) self._exception_callback( SampleException( "unknown data found in chunk %s from %d to %d" % (chunk[1:32], last_index, index))) # stop processing this sio block, it is bad break if data_match: log.debug('Found data match in chunk %s at index %d', chunk[1:32], index) # particle-ize the data block received, return the record # pre-pend the sio header timestamp to the data record (in header_match.group(3)) sample = self._extract_sample( PhsenParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 index += len(data_match.group(0)) last_index = index elif control_match: log.debug( 'Found control match in chunk %s at index %d', chunk[1:32], index) # particle-ize the data block received, return the record # pre-pend the sio header timestamp to the control record (in header_match.group(3)) sample = self._extract_sample( PhsenControlDataParticle, None, header_match.group(3) + control_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 index += len(control_match.group(0)) last_index = index elif chunk[index] == SIO_END: # found end of sio block marker, we are done with this chunk break else: # we found extra data, warn on chunks of extra data not each byte index += 1 self._chunk_sample_count.append(sample_count) # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: sio_header_match = SIO_HEADER_MATCHER.match(chunk) log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) != 'WE': log.warn(" chunk did not match header WE %s", chunk[0:SIO_HEADER_BYTES]) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search( chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1]) if not e_header_match: # no e header match log.warn("*BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback(UnexpectedDataException("Found unexpected data.")) # get the next chunk (timestamp, chunk) = self._chunker.get_next_data() continue # jump to next iteration of the chunk loop payload = chunk[SIO_HEADER_BYTES+HEADER_BYTES+1:] data_split = self.we_split_function(payload) log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0, len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] log.debug('Extracted E Record to store in particle %s', hexlify(e_record)) # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestamp_s = float(fields[0]) timestamp = ntplib.system_to_ntp_time(timestamp_s) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: # create particle log.debug('Particle created with raw data %s', hexlify(e_record)) log.debug('Particle timestamp = %f', timestamp) sample = self._extract_sample(DostaLnWfpSioDataParticle, None, e_record, timestamp) result_particles.append(sample) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match.group(SIO_HEADER_GROUP_ID) == 'AD': log.debug("matched chunk header %s", chunk[1:32]) # start search and match after header data_fail_match = DATA_FAIL_MATCHER.search(chunk[SIO_HEADER_BYTES:]) data_wrapper_match = DATA_WRAPPER_MATCHER.match(chunk[SIO_HEADER_BYTES:]) if data_fail_match: # this ignores any invalid sample data prior to the error tag msg = "Found adcps error type %s exception %s" % (data_fail_match.group(1), data_fail_match.group(2)) log.warn(msg) self._exception_callback(RecoverableSampleException(msg)) elif data_wrapper_match: calculated_xml_checksum = AdcpsJlnSioParser.calc_xml_checksum(data_wrapper_match.group(2)) xml_checksum = int(data_wrapper_match.group(1), 16) if calculated_xml_checksum == xml_checksum: data_match = DATA_MATCHER.search(data_wrapper_match.group(2)) if data_match: log.debug('Found data match in chunk %s', chunk[1:32]) # particle-ize the data block received, return the record sample = self._extract_sample(AdcpsJlnSioDataParticle, None, header_match.group(SIO_HEADER_GROUP_TIMESTAMP) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) else: msg = "Matched adcps xml wrapper but not inside data %s" % \ binascii.hexlify(data_wrapper_match.group(0)) log.warn(msg) self._exception_callback(RecoverableSampleException(msg)) else: msg = "Xml checksum %s does not match calculated %s" % (xml_checksum, calculated_xml_checksum) log.warn(msg) self._exception_callback(RecoverableSampleException(msg)) else: msg = "Unexpected data found within header %s: 0x%s" % (chunk[1:32], binascii.hexlify(chunk)) log.warning(msg) self._exception_callback(UnexpectedDataException(msg)) (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 # Check to see if we are dealing with a wfp_eng SIO chunk if header_match.group(1) == 'WE': data_match = DATA_MATCHER.match(chunk[len(header_match.group(0)):]) controller_timestamp = header_match.group(3) if data_match: sensor_profile_start_time_data = data_match.group(2) if sensor_profile_start_time_data: # Need to unpack the sensor profile start time timestamp fields_prof = struct.unpack_from('>I', sensor_profile_start_time_data[4:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStartTimeParticle, None, controller_timestamp + sensor_profile_start_time_data, float(ntplib.system_to_ntp_time(timestamp))) if sample: log.debug("Sample found: %s", sample) # create particle result_particles.append(sample) sample_count += 1 profile_eng_data = data_match.group(3) # Start from the end of the chunk and working backwards parse_end_point = len(profile_eng_data) # We are going to go through the file data in reverse order since we have a # variable length sample record that could have a decimation factor. # While we do not hit the beginning of the file contents, continue while parse_end_point > 0: # Create the different start indices for the three different scenarios start_index_augmented = parse_end_point-STATUS_BYTES_AUGMENTED start_index_normal = parse_end_point-STATUS_BYTES global_recovered_eng_rec_index = parse_end_point-WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_BYTES # Check for an an augmented status first if start_index_augmented >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_augmented:parse_end_point]): log.debug("Found OffloadProfileData with decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_augmented+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStatusParticle, None, controller_timestamp + profile_eng_data[start_index_augmented:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = start_index_augmented # Check for a normal status elif start_index_normal >= 0 and \ STATUS_START_MATCHER.match(profile_eng_data[start_index_normal:parse_end_point]): log.debug("Found OffloadProfileData without decimation factor") fields_prof = struct.unpack_from('>I', profile_eng_data[start_index_normal+8:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataStatusParticle, None, controller_timestamp + profile_eng_data[start_index_normal:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) parse_end_point = start_index_normal # If neither, we are dealing with a global wfp e recovered engineering data record, # so we will save the start and end points elif global_recovered_eng_rec_index >= 0: log.debug("Found OffloadEngineeringData") fields_prof = struct.unpack_from('>I', profile_eng_data[global_recovered_eng_rec_index:]) timestamp = fields_prof[0] sample = self._extract_sample(WfpEngWfpSioMuleParserDataEngineeringParticle, None, profile_eng_data[ global_recovered_eng_rec_index:parse_end_point], float(ntplib.system_to_ntp_time(timestamp))) # Set the new end point parse_end_point = global_recovered_eng_rec_index # We must not have a good file, log some debug info for now else: log.debug("start_index_augmented %d", start_index_augmented) log.debug("start_index_normal %d", start_index_normal) log.debug("global_recovered_eng_rec_index %d", global_recovered_eng_rec_index) self._exception_callback(SampleException("Data invalid")) if sample: log.debug("Sample found: %s", sample) # create particle result_particles.append(sample) sample_count += 1 else: self._exception_callback(SampleException("Unexpected data")) # Important: We must set the sample count for every chunk self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 if header_match.group(SIO_HEADER_GROUP_ID) == 'AD': log.debug("matched chunk header %s", chunk[1:32]) # start after sio header chunk_idx = SIO_HEADER_BYTES end_idx_okay = SIO_HEADER_BYTES while chunk_idx <= len(chunk): data_fail_match = DATA_FAIL_MATCHER.match(chunk[chunk_idx:]) data_wrapper_match = DATA_WRAPPER_MATCHER.match(chunk[chunk_idx:]) if data_wrapper_match: calculated_xml_checksum = AdcpsJlnSioParser.calc_xml_checksum(data_wrapper_match.group(2)) xml_checksum = int(data_wrapper_match.group(1), 16) if calculated_xml_checksum == xml_checksum: data_match = DATA_MATCHER.search(data_wrapper_match.group(2)) if data_match: if end_idx_okay != chunk_idx: log.info("Unexpected data found from index %d to %d, %s", end_idx_okay, chunk_idx, chunk[end_idx_okay:chunk_idx]) self._exception_callback(UnexpectedDataException("Unexpected data found %s" % chunk[end_idx_okay:chunk_idx])) log.debug('Found data match in chunk %s', chunk[1:32]) # particle-ize the data block received, return the record sample = self._extract_sample(AdcpsJlnSioDataParticle, None, header_match.group(SIO_HEADER_GROUP_TIMESTAMP) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 else: log.warn("Matched adcps xml wrapper but not inside data %s", binascii.hexlify(data_wrapper_match.group(0))) self._exception_callback(RecoverableSampleException( "Matched adcps xml wrapper but not data inside %s" % binascii.hexlify(data_wrapper_match.group(0)))) else: log.warn("Xml checksum %s does not match calculated %s", xml_checksum, calculated_xml_checksum) self._exception_callback(RecoverableSampleException( "Xml checksum %s does not match calculated %s" % (xml_checksum, calculated_xml_checksum))) chunk_idx += len(data_wrapper_match.group(0)) end_idx_okay = chunk_idx elif data_fail_match: # we found an adcps failure message, no data in here just an error if end_idx_okay != chunk_idx: log.warn("Unexpected data found from index %d to %d, %s", end_idx_okay, chunk_idx, binascii.hexlify(chunk[end_idx_okay:chunk_idx])) self._exception_callback(UnexpectedDataException("Unexpected data found %s" % binascii.hexlify(chunk[end_idx_okay:chunk_idx]))) log.warn("Found adcps error type %s exception %s", data_fail_match.group(1), data_fail_match.group(2)) self._exception_callback(RecoverableSampleException("Found adcps error type %s exception %s" % (data_fail_match.group(1), data_fail_match.group(2)))) chunk_idx += len(data_fail_match.group(0)) end_idx_okay = chunk_idx else: # if we have to skip bytes, we have unexplained data chunk_idx += 1 (timestamp, chunk) = self._chunker.get_next_data() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() sample_count = 0 prev_sample = None while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 prev_sample = None # start looping at the end of the header, after \x02 chunk_idx = SIO_HEADER_BYTES + 1 # last possible chunk index, excluding \x03 last_chunk_idx = len(chunk) - 1 if header_match.group(1) == 'CT': log.debug("matched CT chunk header %s", chunk[1:SIO_HEADER_BYTES]) while chunk_idx < last_chunk_idx: data_match = DATA_MATCHER.match(chunk[chunk_idx:chunk_idx+DATA_SAMPLE_BYTES]) if data_match: if self.compare_inductive_id(data_match): # particle-ize the data block received, return the record sample = self._extract_sample(CtdmoParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 chunk_idx += DATA_SAMPLE_BYTES else: log.error('unknown data found in CT chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), chunk_idx) self._exception_callback(SampleException( 'unknown data found in CT chunk at %d, leaving out the rest' % chunk_idx)) break elif header_match.group(1) == 'CO': log.debug("matched CO chunk header %s", chunk[1:SIO_HEADER_BYTES]) while chunk_idx < last_chunk_idx: data_match = CO_MATCHER.match(chunk[chunk_idx:chunk_idx+CO_SAMPLE_BYTES]) if data_match: if self.compare_inductive_id(data_match): # particle-ize the data block received, return the record sample = self._extract_sample(CtdmoOffsetDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 chunk_idx += CO_SAMPLE_BYTES else: log.error('unknown data found in CO chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), chunk_idx) self._exception_callback(SampleException( 'unknown data found in CO chunk at %d, leaving out the rest' % chunk_idx)) break # keep track of how many samples were found in this chunk self._chunk_sample_count.append(sample_count) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match.group(1) == 'PH': # start after the sio header index = header_match.end(0) last_index = index chunk_len = len(chunk) while index < chunk_len: data_match = DATA_MATCHER.match(chunk[index:]) control_match = CONTROL_MATCHER.match(chunk[index:]) # check for any valid match and make sure no extra data was found between valid matches if data_match or control_match or chunk[ index] == SIO_BLOCK_END: # if the indices don't match we have data that doesn't match # exclude the expected possible ph end bytes if last_index != index and chunk[ last_index:index] != PH_EXTRA_END: # we found bad data, send a sample exception but keep processing the file msg = "unknown data found in chunk %s from %d to %d" % ( chunk[1:32], last_index, index) log.warning(msg) self._exception_callback(SampleException(msg)) # stop processing this sio block, it is bad break if data_match: log.debug('Found data match in chunk %s at index %d', chunk[1:32], index) # particle-ize the data block received, return the record sample = self._extract_sample( PhsenAbcdefSioDataParticle, None, (header_match.group(3), data_match), None) result_particles.append(sample) index += len(data_match.group(0)) last_index = index elif control_match: log.debug( 'Found control match in chunk %s at index %d', chunk[1:32], index) # particle-ize the data block received, return the record sample = self._extract_sample( PhsenAbcdefSioControlDataParticle, None, (header_match.group(3), control_match), None) result_particles.append(sample) index += len(control_match.group(0)) last_index = index elif chunk[index] == SIO_BLOCK_END: # found end of sio block marker, we are done with this chunk break else: # we found extra data, warn on chunks of extra data not each byte index += 1 else: # we found unexpected sio instrument id msg = "Unexpected sio instrument header ID %s" % header_match.group( 1) log.warning(msg) self._exception_callback(UnexpectedDataException(msg)) # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() sample_count = 0 prev_sample = None while (chunk != None): header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 prev_sample = None if header_match.group(1) == 'CT': log.debug("matched CT chunk header %s", chunk[1:32]) for data_match in DATA_MATCHER.finditer(chunk): # check if the end of the last sample connects to the start of the next sample if prev_sample is not None: if data_match.start(0) != prev_sample: log.error('extra data found between samples, leaving out the rest of this chunk') break prev_sample = data_match.end(0) # the inductive id is part of the data, check that it matches asciihex_id = binascii.b2a_hex(data_match.group(0)[0:1]) induct_id = int(asciihex_id, 16) if induct_id == self._config.get('inductive_id'): # particle-ize the data block received, return the record sample = self._extract_sample(CtdmoParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 elif header_match.group(1) == 'CO': log.debug("matched CO chunk header %s", chunk[1:32]) for data_match in CO_MATCHER.finditer(chunk): # check if the end of the last sample connects to the start of the next sample if prev_sample is not None: if data_match.start(0) != prev_sample: log.error('extra data found between samples, leaving out the rest of this chunk') break prev_sample = data_match.end(0) # the inductive id is part of the data, check that it matches asciihex_id = binascii.b2a_hex(data_match.group(0)[0:1]) induct_id = int(asciihex_id, 16) if induct_id == self._config.get('inductive_id'): # particle-ize the data block received, return the record sample = self._extract_sample(CtdmoOffsetParserDataParticle, None, header_match.group(3) + data_match.group(0), None) if sample: # create particle result_particles.append(sample) sample_count += 1 # keep track of how many samples were found in this chunk self._chunk_sample_count.append(sample_count) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() while chunk != None: header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 if header_match.group(1) == "PH": # start after the sio header index = header_match.end(0) last_index = index chunk_len = len(chunk) while index < chunk_len: data_match = DATA_MATCHER.match(chunk[index:]) control_match = CONTROL_MATCHER.match(chunk[index:]) # check for any valid match and make sure no extra data was found between valid matches if data_match or control_match or chunk[index] == SIO_END: # if the indices don't match we have data that doesn't match # exclude the expected possible ph end bytes if last_index != index and chunk[last_index:index] != PH_EXTRA_END: # we found bad data, send a sample exception but keep processing the file log.warning("unknown data found in chunk %s from %d to %d", chunk[1:32], last_index, index) self._exception_callback( SampleException( "unknown data found in chunk %s from %d to %d" % (chunk[1:32], last_index, index) ) ) # stop processing this sio block, it is bad break if data_match: log.debug("Found data match in chunk %s at index %d", chunk[1:32], index) # particle-ize the data block received, return the record # pre-pend the sio header timestamp to the data record (in header_match.group(3)) sample = self._extract_sample( PhsenParserDataParticle, None, header_match.group(3) + data_match.group(0), None ) if sample: # create particle result_particles.append(sample) sample_count += 1 index += len(data_match.group(0)) last_index = index elif control_match: log.debug("Found control match in chunk %s at index %d", chunk[1:32], index) # particle-ize the data block received, return the record # pre-pend the sio header timestamp to the control record (in header_match.group(3)) sample = self._extract_sample( PhsenControlDataParticle, None, header_match.group(3) + control_match.group(0), None ) if sample: # create particle result_particles.append(sample) sample_count += 1 index += len(control_match.group(0)) last_index = index elif chunk[index] == SIO_END: # found end of sio block marker, we are done with this chunk break else: # we found extra data, warn on chunks of extra data not each byte index += 1 self._chunk_sample_count.append(sample_count) # non-data does not need to be handled here because for the single file # the data may be corrected and re-written later, it is just ignored until it matches (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list if nothing was parsed. """ result_particles = [] (timestamp, chunk) = self._chunker.get_next_data() while (chunk != None): sio_header_match = SIO_HEADER_MATCHER.match(chunk) sample_count = 0 log.debug('parsing header %s', sio_header_match.group(0)[1:SIO_HEADER_BYTES]) if sio_header_match.group(1) == 'WE': log.trace("********************************matched chunk header %s", chunk[0:SIO_HEADER_BYTES]) # Parse/match the E file header e_header_match = WFP_E_GLOBAL_FLAGS_HEADER_MATCHER.search(chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1]) if e_header_match: log.debug('******************************* HEADER MATCH WAS:') log.debug('%s', ":".join("{:02x}".format(ord(c)) for c in chunk[SIO_HEADER_BYTES:SIO_HEADER_BYTES+HEADER_BYTES+1])) payload = chunk[SIO_HEADER_BYTES+HEADER_BYTES+1:] data_split = self.we_split_function(payload) if data_split: log.debug('Found data match in chunk %s', chunk[1:SIO_HEADER_BYTES]) for ii in range(0,len(data_split)): e_record = payload[data_split[ii][0]:data_split[ii][1]] # particle-ize the data block received, return the record if not STATUS_START_MATCHER.match(e_record[0:STATUS_BYTES]): fields = struct.unpack('>I', e_record[0:4]) timestampS = float(fields[0]) timestamp = ntplib.system_to_ntp_time(timestampS) if len(e_record) == E_GLOBAL_SAMPLE_BYTES: sample = self._extract_sample(DostaLnWfpSioMuleParserDataParticle, None, e_record, timestamp) if sample: # create particle result_particles.append(sample) sample_count += 1 else: # no e header match log.warn("*****************************************************BAD E HEADER 0x%s", ":".join("{:02x}".format(ord(c)) for c in chunk)) self._exception_callback(UnexpectedDataException("Found unexpected data.")) self._chunk_sample_count.append(sample_count) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() return result_particles
def parse_chunks(self): """ Parse chunks for the Telemetered parser. Parse out any pending data chunks in the chunker. If it is a valid data piece, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) while chunk is not None: header_match = SIO_HEADER_MATCHER.match(chunk) if header_match: header_timestamp = header_match.group( SIO_HEADER_GROUP_TIMESTAMP) # start looping at the end of the header chunk_idx = header_match.end(0) if header_match.group(SIO_HEADER_GROUP_ID) == ID_INSTRUMENT: # # Parse the CT record, up to but not including the end of SIO block. # particles = self.parse_ct_record(chunk[chunk_idx:-1], header_timestamp) result_particles.extend(particles) elif header_match.group(SIO_HEADER_GROUP_ID) == ID_OFFSET: (particles, had_error) = parse_co_data( CtdmoGhqrSioTelemeteredOffsetDataParticle, chunk[chunk_idx:-1], header_timestamp, self._extract_sample) if had_error[0]: log.error( 'unknown data found in CO chunk %s at %d, leaving out the rest', binascii.b2a_hex(chunk), had_error[1]) self._exception_callback( SampleException( 'unknown data found in CO chunk at %d, leaving out the rest' % had_error[1])) result_particles.extend(particles) else: message = 'Unexpected Sio Header ID %s' % header_match.group( SIO_HEADER_GROUP_ID) log.warn(message) self._exception_callback(UnexpectedDataException(message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index() self.handle_non_data(non_data, non_end, start) return result_particles