def parse_file(self): """ Parse out any pending data chunks in the chunker. If it is valid data, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. """ for line in self.stream_handle: # If this is a valid sensor data record, # use the extracted fields to generate a particle. sensor_match = SENSOR_DATA_MATCHER.match(line) if sensor_match is not None: particle = self._extract_sample(CtdbpCdefCpInstrumentDataParticle, None, line, None) if particle is not None: self._record_buffer.append(particle) # It's not a sensor data record, see if it's a header record. else: # If it's a valid header record, ignore it. # Otherwise generate warning for unknown data. header_match = HEADER_MATCHER.match(line) log.debug("Header match: %s", str(header_match)) if header_match is None: error_message = 'Unknown data found in chunk %s' % line log.warn(error_message) self._exception_callback(UnexpectedDataException(error_message))
def parse_file(self): """ Parse out any pending data chunks in the chunker. If it is valid data, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. """ for line in self.stream_handle: # If this is a valid sensor data record, # use the extracted fields to generate a particle. sensor_match = SENSOR_DATA_MATCHER.match(line) if sensor_match is not None: particle = self._extract_sample( CtdbpCdefCpInstrumentDataParticle, None, line, None) if particle is not None: self._record_buffer.append(particle) # It's not a sensor data record, see if it's a header record. else: # If it's a valid header record, ignore it. # Otherwise generate warning for unknown data. header_match = HEADER_MATCHER.match(line) log.debug("Header match: %s", str(header_match)) if header_match is None: error_message = 'Unknown data found in chunk %s' % line log.warn(error_message) self._exception_callback( UnexpectedDataException(error_message))
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is valid data, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) while chunk is not None: # If this is a valid sensor data record, # use the extracted fields to generate data particles. sensor_match = SENSOR_DATA_MATCHER.match(chunk) if sensor_match is not None: # First extract the ctdbp_cdef_ce_instrument_recovered particle data_particle = self._extract_sample( CtdbpCdefCeInstrumentDataParticle, None, chunk, None) if data_particle is not None: result_particles.append((data_particle, None)) # Then extract the ctdbp_cdef_ce_dosta_recovered particle dosta_particle = self._extract_sample( CtdbpCdefCeDostaDataParticle, None, chunk, None) if dosta_particle is not None: result_particles.append((dosta_particle, None)) # It's not a sensor data record, see if it's a header record. else: # If it's a valid header record, ignore it. # Otherwise generate warning for unknown data. header_match = HEADER_MATCHER.match(chunk) log.debug("Header match: %s", str(header_match)) if header_match is None: error_message = 'Unknown data found in chunk %s' % chunk log.warn(error_message) self._exception_callback( UnexpectedDataException(error_message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) return result_particles
def parse_chunks(self): """ Parse out any pending data chunks in the chunker. If it is valid data, build a particle. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing. """ result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) while chunk is not None: # If this is a valid sensor data record, # use the extracted fields to generate data particles. sensor_match = SENSOR_DATA_MATCHER.match(chunk) if sensor_match is not None: # First extract the ctdbp_cdef_ce_instrument_recovered particle data_particle = self._extract_sample(CtdbpCdefCeInstrumentDataParticle, None, chunk, None) if data_particle is not None: result_particles.append((data_particle, None)) # Then extract the ctdbp_cdef_ce_dosta_recovered particle dosta_particle = self._extract_sample(CtdbpCdefCeDostaDataParticle, None, chunk, None) if dosta_particle is not None: result_particles.append((dosta_particle, None)) # It's not a sensor data record, see if it's a header record. else: # If it's a valid header record, ignore it. # Otherwise generate warning for unknown data. header_match = HEADER_MATCHER.match(chunk) log.debug("Header match: %s", str(header_match)) if header_match is None: error_message = 'Unknown data found in chunk %s' % chunk log.warn(error_message) self._exception_callback(UnexpectedDataException(error_message)) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) return result_particles