def parse_chunks(self): """ This method parses out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ log.debug("parse_chunks") result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) while chunk is not None: data_match = WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER.match( chunk) if data_match: # Let's first get the 32-bit unsigned int timestamp which should be in the first match group fields_prof = struct.unpack_from('>I', data_match.group(1)) timestamp = fields_prof[0] self._timestamp = float(ntplib.system_to_ntp_time(timestamp)) # particle-ize the data block received, return the record sample = self._extract_sample( self._particle_class, None, chunk, internal_timestamp=self._timestamp) if sample: # create particle log.trace( "Extracting sample chunk 0x%s with read_state: %s", binascii.b2a_hex(chunk), self._read_state) self._increment_state(len(chunk)) result_particles.append( (sample, copy.copy(self._read_state))) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) return result_particles
def _build_parsed_values(self): """ Take something in the data format and turn it into an array of dictionaries defining the data in the particle with the appropriate tag. @throws SampleException If there is a problem with sample creation """ match = WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER.match( self.raw_data) if not match: raise SampleException( "DostaLnWfpInstrumentParserDataParticle: No regex match of parsed sample data: [0x%s]", binascii.b2a_hex(self.raw_data)) try: # Let's first get the 32-bit unsigned int timestamp which should be in the first match group fields_prof = struct.unpack_from('>I', match.group(1)) wfp_timestamp = fields_prof[0] # Now let's grab the global engineering data record match group # Should be 5 float 32-bit values followed by 3 unsigned int 16-bit values fields_prof = struct.unpack_from('>fffffHHH', match.group(2)) # the optode_oxygen field should be the 4th value estimated_oxygen_concentration = fields_prof[3] # the optode_oxygen field should be the 5th value optode_temperature = fields_prof[4] except (ValueError, TypeError, IndexError) as ex: raise SampleException( "Error (%s) while decoding parameters in data: [0x%s]" % (ex, binascii.b2a_hex(match.group(0)))) result = [ self._encode_value( DostaLnWfpInstrumentParserDataParticleKey. ESTIMATED_OXYGEN_CONCENTRATION, estimated_oxygen_concentration, float), self._encode_value( DostaLnWfpInstrumentParserDataParticleKey.OPTODE_TEMPERATURE, optode_temperature, float), self._encode_value( DostaLnWfpInstrumentParserDataParticleKey.WFP_TIMESTAMP, wfp_timestamp, int) ] return result
def _build_parsed_values(self): """ Take something in the data format and turn it into an array of dictionaries defining the data in the particle with the appropriate tag. @throws SampleException If there is a problem with sample creation """ # NOTE: since we are dropping the status messages in the sieve, only # sampes should make it here if len(self.raw_data) != E_GLOBAL_SAMPLE_BYTES: raise SampleException( "Error (%s) while decoding parameters in data: [%s]" % (ex, match.group(0))) else: try: match = WFP_E_SAMPLE_MATCHER.match(self.raw_data) # grab the timestamp from the first match group fields_prof = struct.unpack('>I', match.group(1)) wfp_timestamp = fields_prof[0] # and parse the rest of the data from the next match group fields_prof = struct.unpack('>f f f f f H H H', match.group(2)) optode_oxygen = fields_prof[3] optode_temperature = fields_prof[4] except (ValueError, TypeError, IndexError) as ex: raise SampleException( "Error (%s) while decoding parameters in data: [%s]" % (ex, match.group(0))) result = [ self._encode_value( DostaLnWfpSioMuleDataParticleKey.OPTODE_OXYGEN, optode_oxygen, float), self._encode_value( DostaLnWfpSioMuleDataParticleKey.OPTODE_TEMPERATURE, optode_temperature, float), self._encode_value( DostaLnWfpSioMuleDataParticleKey.WFP_TIMESTAMP, wfp_timestamp, int) ] log.debug('DostLnWfpSioMuleDataParticle: particle=%s', result) return result
def parse_chunks(self): """ This method parses out any pending data chunks in the chunker. If it is a valid data piece, build a particle, update the position and timestamp. Go until the chunker has no more valid data. @retval a list of tuples with sample particles encountered in this parsing, plus the state. An empty list of nothing was parsed. """ log.debug("parse_chunks") result_particles = [] (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) while chunk is not None: data_match = WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER.match(chunk) if data_match: # Let's first get the 32-bit unsigned int timestamp which should be in the first match group fields_prof = struct.unpack_from('>I', data_match.group(1)) timestamp = fields_prof[0] self._timestamp = float(ntplib.system_to_ntp_time(timestamp)) # particle-ize the data block received, return the record sample = self._extract_sample(self._particle_class, None, chunk, internal_timestamp=self._timestamp) if sample: # create particle log.trace("Extracting sample chunk 0x%s with read_state: %s", binascii.b2a_hex(chunk), self._read_state) self._increment_state(len(chunk)) result_particles.append((sample, copy.copy(self._read_state))) (nd_timestamp, non_data, non_start, non_end) = self._chunker.get_next_non_data_with_index(clean=False) (timestamp, chunk, start, end) = self._chunker.get_next_data_with_index(clean=True) self.handle_non_data(non_data, non_end, start) return result_particles
def _build_parsed_values(self): """ Take something in the binary data values and turn it into a particle with the appropriate tag. throws SampleException If there is a problem with sample creation """ match = WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER.match(self.raw_data) if not match: raise RecoverableSampleException( "WfpEngWfpSioMuleParserDataEngineeringParticle: No regex match of parsed sample data: [%s]", self.raw_data) try: # Let's first get the 32-bit unsigned int timestamp which should be in the first match group fields = struct.unpack_from('>I', match.group(1)) wfp_timestamp = fields[0] # Now let's grab the global engineering data record match group # Should be 3 float 32-bit values fields = struct.unpack_from('>fff', match.group(2)) current = fields[0] voltage = fields[1] pressure = fields[2] result = [self._encode_value(WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_TIMESTAMP, wfp_timestamp, int), self._encode_value(WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_CURRENT, current, float), self._encode_value(WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_VOLTAGE, voltage, float), self._encode_value(WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_PRESSURE, pressure, float)] except (ValueError, TypeError, IndexError) as ex: raise RecoverableSampleException("Error (%s) while decoding parameters in data: [%s]" % (ex, match.group(0))) return result
def _build_parsed_values(self): """ Take something in the data format and turn it into an array of dictionaries defining the data in the particle with the appropriate tag. @throws SampleException If there is a problem with sample creation """ # NOTE: since we are dropping the status messages in the sieve, only # samples should make it here if len(self.raw_data) != E_GLOBAL_SAMPLE_BYTES: raise SampleException("Error while decoding parameters in data: ") try: match = WFP_E_SAMPLE_MATCHER.match(self.raw_data) # grab the timestamp from the first match group fields_prof = struct.unpack('>I', match.group(1)) wfp_timestamp = fields_prof[0] # and parse the rest of the data from the next match group fields_prof = struct.unpack('>f f f f f H H H', match.group(2)) optode_oxygen = fields_prof[3] optode_temperature = fields_prof[4] except (ValueError, TypeError, IndexError) as ex: raise SampleException("Error (%s) while decoding parameters in data: [%s]" % (ex, match.group(0))) result = [self._encode_value(DostaLnWfpSioDataParticleKey.OPTODE_OXYGEN, optode_oxygen, float), self._encode_value(DostaLnWfpSioDataParticleKey.OPTODE_TEMPERATURE, optode_temperature, float), self._encode_value(DostaLnWfpSioDataParticleKey.WFP_TIMESTAMP, wfp_timestamp, int)] log.debug('DostLnWfpSioMuleDataParticle: particle=%s', result) return result
def _build_parsed_values(self): """ Take something in the data format and turn it into an array of dictionaries defining the data in the particle with the appropriate tag. @throws SampleException If there is a problem with sample creation """ match = WFP_E_GLOBAL_RECOVERED_ENG_DATA_SAMPLE_MATCHER.match(self.raw_data) if not match: raise SampleException("DostaLnWfpInstrumentParserDataParticle: No regex match of parsed sample data: [0x%s]", binascii.b2a_hex(self.raw_data)) try: # Let's first get the 32-bit unsigned int timestamp which should be in the first match group fields_prof = struct.unpack_from('>I', match.group(1)) wfp_timestamp = fields_prof[0] # Now let's grab the global engineering data record match group # Should be 5 float 32-bit values followed by 3 unsigned int 16-bit values fields_prof = struct.unpack_from('>fffffHHH', match.group(2)) # the optode_oxygen field should be the 4th value estimated_oxygen_concentration = fields_prof[3] # the optode_oxygen field should be the 5th value optode_temperature = fields_prof[4] except (ValueError, TypeError, IndexError) as ex: raise SampleException("Error (%s) while decoding parameters in data: [0x%s]" % (ex, binascii.b2a_hex(match.group(0)))) result = [self._encode_value(DostaLnWfpInstrumentParserDataParticleKey.ESTIMATED_OXYGEN_CONCENTRATION, estimated_oxygen_concentration, float), self._encode_value(DostaLnWfpInstrumentParserDataParticleKey.OPTODE_TEMPERATURE, optode_temperature, float), self._encode_value(DostaLnWfpInstrumentParserDataParticleKey.WFP_TIMESTAMP, wfp_timestamp, int)] return result