def _generate_internal_timestamp(record_dict): """ Generates the internal timestamp from the given DCL Controller Timestamp. :param record_dict: dictionary containing the dcl controller timestamp str parameter :return: the internal timestamp """ return float(dcl_controller_timestamp_to_ntp_time( record_dict[Pco2wAbcDataParticleKey.DCL_CONTROLLER_TIMESTAMP]))
def parse_file(self): """ Parse the zplsc_c log file (averaged condensed data). Read file line by line. Values are extracted from lines containing condensed ASCII data @return: dictionary of data values with the particle names as keys or None """ # Loop over all lines in the data file and parse the data to generate particles for number, line in enumerate(self._stream_handle, start=1): # Check if this is the dcl status log match = DCL_LOG_MATCHER.match(line) if match is not None: log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number, match.groups()) # No data to extract, move on to the next line continue # Check if this is the instrument phase status log match = PHASE_STATUS_MATCHER.match(line) if match is not None: log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number, match.groups()) # No data to extract, move on to the next line continue # Check if this is the instrument condensed ASCII data match = SENSOR_DATA_MATCHER.match(line) if match is not None: log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number, match.groups()) # Extract the condensed ASCII data from this line data_dict = self.parse_line(match) if data_dict is None: log.error('Erroneous data found in line %s: %s', number, line) continue # Convert the DCL timestamp into the particle timestamp time_stamp = dcl_controller_timestamp_to_ntp_time( match.group('dcl_timestamp')) # Extract a particle and append it to the record buffer particle = self._extract_sample(ZplscCInstrumentDataParticle, None, data_dict, time_stamp) if particle is not None: log.trace('Parsed particle: %s' % particle.generate_dict()) self._record_buffer.append(particle) continue # Error, line did not match any expected regex self._exception_callback( RecoverableSampleException('Unknown data found in line %s:%s' % (number, line)))
def _generate_internal_timestamp(record_dict): """ Generates the internal timestamp from the given DCL Controller Timestamp. :param record_dict: dictionary containing the dcl controller timestamp str parameter :return: the internal timestamp """ return float( dcl_controller_timestamp_to_ntp_time( record_dict[Pco2wAbcDataParticleKey.DCL_CONTROLLER_TIMESTAMP]))
def __init__(self, raw_data, instrument_particle_map, *args, **kwargs): super(DclInstrumentDataParticle, self).__init__(raw_data, *args, **kwargs) # The particle timestamp is the DCL Controller timestamp. # Convert the DCL controller timestamp string to NTP time (in seconds and microseconds). dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP] elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp) self.set_internal_timestamp(elapsed_seconds_useconds) self.instrument_particle_map = instrument_particle_map
def _extract_instrument_ntp_timestamp(self, inst_match): """ This function will create a timestamp to be used as the internal timestamp for the instrument particle is generated. """ # calculate the instrument particle internal timestamp # from the DCL timestamp. return dcl_controller_timestamp_to_ntp_time(inst_match.group( InstrumentDataMatchGroups.INST_GROUP_DCL_TIMESTAMP))
def _extract_instrument_ntp_timestamp(self, inst_match): """ This function will create a timestamp to be used as the internal timestamp for the instrument particle is generated. """ # calculate the instrument particle internal timestamp # from the DCL timestamp. return dcl_controller_timestamp_to_ntp_time(inst_match.group( InstrumentDataMatchGroups.INST_GROUP_DCL_TIMESTAMP))
def parse_file(self): """ Parse the zplsc_c log file (averaged condensed data). Read file line by line. Values are extracted from lines containing condensed ASCII data @return: dictionary of data values with the particle names as keys or None """ # Loop over all lines in the data file and parse the data to generate particles for number, line in enumerate(self._stream_handle, start=1): # Check if this is the dcl status log match = DCL_LOG_MATCHER.match(line) if match is not None: log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number, match.groups()) # No data to extract, move on to the next line continue # Check if this is the instrument phase status log match = PHASE_STATUS_MATCHER.match(line) if match is not None: log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number, match.groups()) # No data to extract, move on to the next line continue # Check if this is the instrument condensed ASCII data match = SENSOR_DATA_MATCHER.match(line) if match is not None: log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number, match.groups()) # Extract the condensed ASCII data from this line data_dict = self.parse_line(match) if data_dict is None: log.error('Erroneous data found in line %s: %s', number, line) continue # Convert the DCL timestamp into the particle timestamp time_stamp = dcl_controller_timestamp_to_ntp_time(match.group('dcl_timestamp')) # Extract a particle and append it to the record buffer particle = self._extract_sample( ZplscCInstrumentDataParticle, None, data_dict, time_stamp) if particle is not None: log.trace('Parsed particle: %s' % particle.generate_dict()) self._record_buffer.append(particle) continue # Error, line did not match any expected regex self._exception_callback( RecoverableSampleException('Unknown data found in line %s:%s' % (number, line)))
def __init__(self, raw_data, port_timestamp=None, internal_timestamp=None, preferred_timestamp=DataParticleKey.PORT_TIMESTAMP, quality_flag=DataParticleValue.OK, new_sequence=None): super(SpkirAbjDclInstrumentDataParticle, self).__init__(raw_data, port_timestamp, internal_timestamp, preferred_timestamp, quality_flag, new_sequence) # The particle timestamp is the DCL Controller timestamp. # Convert the DCL controller timestamp string to NTP time (in seconds and microseconds). dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP] elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time( dcl_controller_timestamp) self.set_internal_timestamp(elapsed_seconds_useconds)
def __init__(self, raw_data, port_timestamp=None, internal_timestamp=None, preferred_timestamp=DataParticleKey.PORT_TIMESTAMP, quality_flag=DataParticleValue.OK, new_sequence=None): super(DostaAbcdjmDclInstrumentDataParticle, self).__init__(raw_data, port_timestamp, internal_timestamp, preferred_timestamp, quality_flag, new_sequence) # The particle timestamp is the DCL Controller timestamp. # Convert the DCL controller timestamp string to NTP time (in seconds and microseconds). dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP] elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp) self.set_internal_timestamp(elapsed_seconds_useconds)
def _build_parsed_values(self): """ Build parsed values for Recovered and Telemetered Instrument Data Particle. """ # Generate a particle by calling encode_value for each entry # in the Instrument Particle Mapping table, # where each entry is a tuple containing the particle field name, # an index into the match groups (which is what has been stored in raw_data), # and a function to use for data conversion. # The particle timestamp is the DCL Controller timestamp. # Convert the DCL controller timestamp string to NTP time (in seconds and microseconds). dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP] elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp) self.set_internal_timestamp(elapsed_seconds_useconds) return [self._encode_value(name, self.raw_data[group], function) for name, group, function in INSTRUMENT_PARTICLE_MAP]
def _build_parsed_values(self): """ Build parsed values for Recovered and Telemetered Instrument Data Particle. """ # Generate a particle by calling encode_value for each entry # in the Instrument Particle Mapping table, # where each entry is a tuple containing the particle field name, # an index into the match groups (which is what has been stored in raw_data), # and a function to use for data conversion. # The particle timestamp is the DCL Controller timestamp. # Convert the DCL controller timestamp string to NTP time (in seconds and microseconds). dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP] elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time( dcl_controller_timestamp) self.set_internal_timestamp(elapsed_seconds_useconds) return [ self._encode_value(name, self.raw_data[group], function) for name, group, function in INSTRUMENT_PARTICLE_MAP ]
def parse_file(self): """ Parse through the file, pulling single lines and comparing to the established patterns, generating particles for data lines """ # initialize data for modem particle first_timestamp = None date_timestamp = None distance = None dsp_bat = None xmit_bat = None # initialize raw_data for CsppEngDclEngDataParticle self._eng_data = [None] * 10 for line in self._stream_handle: data_match = RECORD_MATCHER.match(line) if data_match is None: message = 'got malformed line %s ' % line log.warn(message) self._exception_callback(RecoverableSampleException(message)) continue if data_match.group('mode') == 'sent': continue # skip sent messages, go to next line timestamp_str = data_match.group('timestamp') message = data_match.group('message') if first_timestamp is None: first_timestamp = timestamp_str # save the first timestamp for the modem particle # save off header information for modem particle # modem particle created after processing entire file. range_match = RANGE_MATCHER.match(message) if range_match: distance = range_match.group('range') continue # go to next line dsp_match = DSP_MATCHER.match(message) if dsp_match: dsp_bat = dsp_match.group('dsp_bat') continue # go to next line xmit_match = XMIT_MATCHER.match(message) if xmit_match: xmit_bat = xmit_match.group('dsp_bat') continue # go to next line # process NMEA sentences nmea_match = NMEA_MATCHER.match(message) if nmea_match: sentence = nmea_match.group('sentence') checksum = int(nmea_match.group('checksum'), 16) # Convert to integer # Note: NMEA checksums typically do not include the $ at the # beginning of the sentence but it appears Wetlabs implemented # it that way. comp_checksum = self.calc_checksum(sentence) if comp_checksum == checksum: fields = sentence.split(',') command = fields[5] count = fields[6] sentence_params = NMEA_SENTENCE_MAP.get(command) if sentence_params is None: # skip NMEA sentences we are not looking for log.debug('NMEA sentence skipped %s', line) continue # go to next line expected_count, particle_class = sentence_params if int(count) != expected_count: message = 'did not get expected number of fields on line %s' % line log.warn(message) self._exception_callback( RecoverableSampleException(message)) continue # go to next line if particle_class == CsppEngDclEngDataParticle: if command == 'DATE': date_timestamp = timestamp_str # save timestamp from the DATE record self.process_date(fields[7:]) elif command == 'PFS': self._eng_data[1:3] = fields[7:9] elif command == 'PST': self.process_start(fields[7:]) elif command == 'ENA': self._eng_data[5:7] = fields[7:9] elif command == 'WHE': self.process_wave(fields[7:]) else: # Create particle and add to buffer timestamp = dcl_controller_timestamp_to_ntp_time( timestamp_str) data_particle = self._extract_sample( particle_class, None, fields[7:], timestamp) self._record_buffer.append(data_particle) else: message = 'checksum failed on line %s' % line log.warn(message) self._exception_callback( RecoverableSampleException(message)) # end for loop # only send modem particle if we have a timestamp # and at least one parameter if first_timestamp and (distance or dsp_bat or xmit_bat): timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp) data_particle = self._extract_sample(CsppEngDclModemParticle, None, [distance, dsp_bat, xmit_bat], timestamp) self._record_buffer.append(data_particle) if any(self._eng_data ): # Publish CsppEngDclEngDataParticle if we have any data if date_timestamp: # preference is DATE timestamp timestamp = dcl_controller_timestamp_to_ntp_time( date_timestamp) else: timestamp = dcl_controller_timestamp_to_ntp_time( first_timestamp) data_particle = self._extract_sample(CsppEngDclEngDataParticle, None, self._eng_data, timestamp) self._record_buffer.append(data_particle)
def _build_parsed_values(self): """ Extracts PHSEN ABCDEF DCL Metadata data from raw_data. @returns result a list of dictionaries of particle data """ ## extract the time from the raw_data tuple dcl_controller_timestamp = self.raw_data[0] ## convert the time converted_time = dcl_controller_timestamp_to_ntp_time( dcl_controller_timestamp) ## set the converted time to the particle internal timestamp self.set_internal_timestamp(converted_time) log.trace( " XX ## XX ## XX ## XX PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time) ## extract the working_record string from the raw data tuple working_record = self.raw_data[1] log.trace( "PhsenAbcdefDclMetadataDataParticle._generate_particle(): dcl_controller_timestamp= %s, " "working_record= %s", dcl_controller_timestamp, working_record) log.trace( "PhsenAbcdefDclMetadataDataParticle._generate_particle(): Data Length= %s, working_record Length= %s", int(working_record[3:5], 16), len(working_record)) ## Per the IDD, voltage_battery data is optional and not guaranteed to be included in every CONTROL ## data record. Nominal size of a metadata string without the voltage_battery data is 39 (including the #). ## Voltage data adds 4 ascii characters to that, so raw_data greater than 41 contains voltage data, ## anything smaller does not. if len(working_record) >= 41: have_voltage_battery_data = True else: have_voltage_battery_data = False log.trace( "PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "raw_data len= %s and contains: %s, have_voltage_battery_data= %s ", len(working_record), working_record, have_voltage_battery_data) ## ## Begin saving particle data ## unique_id_ascii_hex = working_record[1:3] ## convert 2 ascii (hex) chars to int unique_id_int = int(unique_id_ascii_hex, 16) record_type_ascii_hex = working_record[5:7] ## convert 2 ascii (hex) chars to int record_type_int = int(record_type_ascii_hex, 16) record_time_ascii_hex = working_record[7:15] ## convert 8 ascii (hex) chars to int record_time_int = int(record_time_ascii_hex, 16) ## FLAGS flags_ascii_hex = working_record[15:19] ## convert 4 ascii (hex) chars to list of binary data flags_ascii_int = int(flags_ascii_hex, 16) binary_list = [(flags_ascii_int >> x) & 0x1 for x in range(16)] # log.debug("PhsenAbcdefDclMetadataDataParticle._generate_particle(): binary_list= %s", binary_list) clock_active = binary_list[0] recording_active = binary_list[1] record_end_on_time = binary_list[2] record_memory_full = binary_list[3] record_end_on_error = binary_list[4] data_download_ok = binary_list[5] flash_memory_open = binary_list[6] battery_low_prestart = binary_list[7] battery_low_measurement = binary_list[8] battery_low_blank = binary_list[9] battery_low_external = binary_list[10] external_device1_fault = binary_list[11] external_device2_fault = binary_list[12] external_device3_fault = binary_list[13] flash_erased = binary_list[14] power_on_invalid = binary_list[15] num_data_records_ascii_hex = working_record[19:25] ## convert 6 ascii (hex) chars to int num_data_records_int = int(num_data_records_ascii_hex, 16) num_error_records_ascii_hex = working_record[25:31] ## convert 6 ascii (hex) chars to int num_error_records_int = int(num_error_records_ascii_hex, 16) num_bytes_stored_ascii_hex = working_record[31:37] ## convert 6 ascii (hex) chars to int num_bytes_stored_int = int(num_bytes_stored_ascii_hex, 16) calculated_checksum = _calculate_working_record_checksum( working_record) ## Record may not have voltage data... if have_voltage_battery_data: voltage_battery_ascii_hex = working_record[37:41] ## convert 4 ascii (hex) chars to int voltage_battery_int = int(voltage_battery_ascii_hex, 16) passed_checksum_ascii_hex = working_record[41:43] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 else: voltage_battery_int = None passed_checksum_ascii_hex = working_record[37:39] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 log.debug( "### ### ###PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int) ## ASSEMBLE THE RESULTANT PARTICLE.. resultant_particle_data = [{ DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.DCL_CONTROLLER_TIMESTAMP, DataParticleKey.VALUE: dcl_controller_timestamp }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.UNIQUE_ID, DataParticleKey.VALUE: unique_id_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_TYPE, DataParticleKey.VALUE: record_type_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_TIME, DataParticleKey.VALUE: record_time_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.CLOCK_ACTIVE, DataParticleKey.VALUE: clock_active }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORDING_ACTIVE, DataParticleKey.VALUE: recording_active }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_TIME, DataParticleKey.VALUE: record_end_on_time }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_MEMORY_FULL, DataParticleKey.VALUE: record_memory_full }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_ERROR, DataParticleKey.VALUE: record_end_on_error }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.DATA_DOWNLOAD_OK, DataParticleKey.VALUE: data_download_ok }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.FLASH_MEMORY_OPEN, DataParticleKey.VALUE: flash_memory_open }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_PRESTART, DataParticleKey.VALUE: battery_low_prestart }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_MEASUREMENT, DataParticleKey.VALUE: battery_low_measurement }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_BLANK, DataParticleKey.VALUE: battery_low_blank }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_EXTERNAL, DataParticleKey.VALUE: battery_low_external }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE1_FAULT, DataParticleKey.VALUE: external_device1_fault }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE2_FAULT, DataParticleKey.VALUE: external_device2_fault }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE3_FAULT, DataParticleKey.VALUE: external_device3_fault }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.FLASH_ERASED, DataParticleKey.VALUE: flash_erased }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.POWER_ON_INVALID, DataParticleKey.VALUE: power_on_invalid }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_DATA_RECORDS, DataParticleKey.VALUE: num_data_records_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_ERROR_RECORDS, DataParticleKey.VALUE: num_error_records_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_BYTES_STORED, DataParticleKey.VALUE: num_bytes_stored_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.VOLTAGE_BATTERY, DataParticleKey.VALUE: voltage_battery_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.PASSED_CHECKSUM, DataParticleKey.VALUE: checksum_final }] log.debug( "PhsenAbcdefDclMetadataDataParticle._build_parsed_values(): resultant_particle_data= %s", resultant_particle_data) return resultant_particle_data
def _build_parsed_values(self): """ Extracts PHSEN ABCDEF DCL Instrument data from the raw_data tuple. @returns result a list of dictionaries of particle data """ ## extract the time from the raw_data touple dcl_controller_timestamp = self.raw_data[0] ## convert the time converted_time = dcl_controller_timestamp_to_ntp_time( dcl_controller_timestamp) ## set the converted time to the particle internal timestamp self.set_internal_timestamp(converted_time) log.debug( " XX ## XX ## XX ## XX PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time) ## extract the working_record string from the raw data tuple working_record = self.raw_data[1] log.trace( "PhsenAbcdefDclInstrumentDataParticle._generate_particle(): dcl_controller_timestamp= %s, " "working_record= %s", dcl_controller_timestamp, working_record) log.trace( "PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "Data Length= %s, working_record Length= %s", int(working_record[3:5], 16), len(working_record)) ## ## Begin saving particle data ## unique_id_ascii_hex = working_record[1:3] ## convert 2 ascii (hex) chars to int unique_id_int = int(unique_id_ascii_hex, 16) record_type_ascii_hex = working_record[5:7] ## convert 2 ascii (hex) chars to int record_type_int = int(record_type_ascii_hex, 16) record_time_ascii_hex = working_record[7:15] ## convert 8 ascii (hex) chars to int record_time_int = int(record_time_ascii_hex, 16) thermistor_start_ascii_hex = working_record[15:19] ## convert 4 ascii (hex) chars to int thermistor_start_int = int(thermistor_start_ascii_hex, 16) reference_light_measurements_list_int = self._create_reference_light_measurements_array( working_record) light_measurements_list_int = self._create_light_measurements_array( working_record) voltage_battery_ascii_hex = working_record[455:459] ## convert 4 ascii (hex) chars to int voltage_battery_int = int(voltage_battery_ascii_hex, 16) thermistor_end_ascii_hex = working_record[459:463] ## convert 4 ascii (hex) chars to int thermistor_end_int = int(thermistor_end_ascii_hex, 16) passed_checksum_ascii_hex = working_record[463:465] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) calculated_checksum = _calculate_working_record_checksum( working_record) log.trace( "### ### ###PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 ## ASSEMBLE THE RESULTANT PARTICLE.. resultant_particle_data = [{ DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.DCL_CONTROLLER_TIMESTAMP, DataParticleKey.VALUE: dcl_controller_timestamp }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.UNIQUE_ID, DataParticleKey.VALUE: unique_id_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TYPE, DataParticleKey.VALUE: record_type_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TIME, DataParticleKey.VALUE: record_time_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_START, DataParticleKey.VALUE: thermistor_start_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey. REFERENCE_LIGHT_MEASUREMENTS, DataParticleKey.VALUE: reference_light_measurements_list_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.LIGHT_MEASUREMENTS, DataParticleKey.VALUE: light_measurements_list_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.VOLTAGE_BATTERY, DataParticleKey.VALUE: voltage_battery_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_END, DataParticleKey.VALUE: thermistor_end_int }, { DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.PASSED_CHECKSUM, DataParticleKey.VALUE: checksum_final }] log.debug( "PhsenAbcdefDclInstrumentDataParticle._build_parsed_values(): resultant_particle_data= %s", resultant_particle_data) return resultant_particle_data
def parse_file(self): """ Parse through the file, pulling single lines and comparing to the established patterns, generating particles for data lines """ # initialize data for modem particle first_timestamp = None date_timestamp = None distance = None dsp_bat = None xmit_bat = None # initialize raw_data for CsppEngDclEngDataParticle self._eng_data = [None] * 10 for line in self._stream_handle: data_match = RECORD_MATCHER.match(line) if data_match is None: message = 'got malformed line %s ' % line log.warn(message) self._exception_callback(RecoverableSampleException(message)) continue if data_match.group('mode') == 'sent': continue # skip sent messages, go to next line timestamp_str = data_match.group('timestamp') message = data_match.group('message') if first_timestamp is None: first_timestamp = timestamp_str # save the first timestamp for the modem particle # save off header information for modem particle # modem particle created after processing entire file. range_match = RANGE_MATCHER.match(message) if range_match: distance = range_match.group('range') continue # go to next line dsp_match = DSP_MATCHER.match(message) if dsp_match: dsp_bat = dsp_match.group('dsp_bat') continue # go to next line xmit_match = XMIT_MATCHER.match(message) if xmit_match: xmit_bat = xmit_match.group('dsp_bat') continue # go to next line # process NMEA sentences nmea_match = NMEA_MATCHER.match(message) if nmea_match: sentence = nmea_match.group('sentence') checksum = int(nmea_match.group('checksum'), 16) # Convert to integer # Note: NMEA checksums typically do not include the $ at the # beginning of the sentence but it appears Wetlabs implemented # it that way. comp_checksum = self.calc_checksum(sentence) if comp_checksum == checksum: fields = sentence.split(',') command = fields[5] count = fields[6] sentence_params = NMEA_SENTENCE_MAP.get(command) if sentence_params is None: # skip NMEA sentences we are not looking for log.debug('NMEA sentence skipped %s', line) continue # go to next line expected_count, particle_class = sentence_params if int(count) != expected_count: message = 'did not get expected number of fields on line %s' % line log.warn(message) self._exception_callback(RecoverableSampleException(message)) continue # go to next line if particle_class == CsppEngDclEngDataParticle: if command == 'DATE': date_timestamp = timestamp_str # save timestamp from the DATE record self.process_date(fields[7:]) elif command == 'PFS': self._eng_data[1:3] = fields[7:9] elif command == 'PST': self.process_start(fields[7:]) elif command == 'ENA': self._eng_data[5:7] = fields[7:9] elif command == 'WHE': self.process_wave(fields[7:]) else: # Create particle and add to buffer timestamp = dcl_controller_timestamp_to_ntp_time(timestamp_str) data_particle = self._extract_sample(particle_class, None, fields[7:], timestamp) self._record_buffer.append(data_particle) else: message = 'checksum failed on line %s' % line log.warn(message) self._exception_callback(RecoverableSampleException(message)) # end for loop # only send modem particle if we have a timestamp # and at least one parameter if first_timestamp and (distance or dsp_bat or xmit_bat): timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp) data_particle = self._extract_sample(CsppEngDclModemParticle, None, [distance, dsp_bat, xmit_bat], timestamp) self._record_buffer.append(data_particle) if any(self._eng_data): # Publish CsppEngDclEngDataParticle if we have any data if date_timestamp: # preference is DATE timestamp timestamp = dcl_controller_timestamp_to_ntp_time(date_timestamp) else: timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp) data_particle = self._extract_sample(CsppEngDclEngDataParticle, None, self._eng_data, timestamp) self._record_buffer.append(data_particle)
def _build_parsed_values(self): """ Extracts PHSEN ABCDEF DCL Metadata data from raw_data. @returns result a list of dictionaries of particle data """ ## extract the time from the raw_data tuple dcl_controller_timestamp = self.raw_data[0] ## convert the time converted_time = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp) ## set the converted time to the particle internal timestamp self.set_internal_timestamp(converted_time) log.trace(" XX ## XX ## XX ## XX PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time) ## extract the working_record string from the raw data tuple working_record = self.raw_data[1] log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): dcl_controller_timestamp= %s, " "working_record= %s", dcl_controller_timestamp, working_record) log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): Data Length= %s, working_record Length= %s", int(working_record[3:5], 16), len(working_record)) ## Per the IDD, voltage_battery data is optional and not guaranteed to be included in every CONTROL ## data record. Nominal size of a metadata string without the voltage_battery data is 39 (including the #). ## Voltage data adds 4 ascii characters to that, so raw_data greater than 41 contains voltage data, ## anything smaller does not. if len(working_record) >= 41: have_voltage_battery_data = True else: have_voltage_battery_data = False log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "raw_data len= %s and contains: %s, have_voltage_battery_data= %s ", len(working_record), working_record, have_voltage_battery_data) ## ## Begin saving particle data ## unique_id_ascii_hex = working_record[1:3] ## convert 2 ascii (hex) chars to int unique_id_int = int(unique_id_ascii_hex, 16) record_type_ascii_hex = working_record[5:7] ## convert 2 ascii (hex) chars to int record_type_int = int(record_type_ascii_hex, 16) record_time_ascii_hex = working_record[7:15] ## convert 8 ascii (hex) chars to int record_time_int = int(record_time_ascii_hex, 16) ## FLAGS flags_ascii_hex = working_record[15:19] ## convert 4 ascii (hex) chars to list of binary data flags_ascii_int = int(flags_ascii_hex, 16) binary_list = [(flags_ascii_int >> x) & 0x1 for x in range(16)] # log.debug("PhsenAbcdefDclMetadataDataParticle._generate_particle(): binary_list= %s", binary_list) clock_active = binary_list[0] recording_active = binary_list[1] record_end_on_time = binary_list[2] record_memory_full = binary_list[3] record_end_on_error = binary_list[4] data_download_ok = binary_list[5] flash_memory_open = binary_list[6] battery_low_prestart = binary_list[7] battery_low_measurement = binary_list[8] battery_low_blank = binary_list[9] battery_low_external = binary_list[10] external_device1_fault = binary_list[11] external_device2_fault = binary_list[12] external_device3_fault = binary_list[13] flash_erased = binary_list[14] power_on_invalid = binary_list[15] num_data_records_ascii_hex = working_record[19:25] ## convert 6 ascii (hex) chars to int num_data_records_int = int(num_data_records_ascii_hex, 16) num_error_records_ascii_hex = working_record[25:31] ## convert 6 ascii (hex) chars to int num_error_records_int = int(num_error_records_ascii_hex, 16) num_bytes_stored_ascii_hex = working_record[31:37] ## convert 6 ascii (hex) chars to int num_bytes_stored_int = int(num_bytes_stored_ascii_hex, 16) calculated_checksum = _calculate_working_record_checksum(working_record) ## Record may not have voltage data... if have_voltage_battery_data: voltage_battery_ascii_hex = working_record[37:41] ## convert 4 ascii (hex) chars to int voltage_battery_int = int(voltage_battery_ascii_hex, 16) passed_checksum_ascii_hex = working_record[41:43] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 else: voltage_battery_int = None passed_checksum_ascii_hex = working_record[37:39] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 log.debug("### ### ###PhsenAbcdefDclMetadataDataParticle._generate_particle(): " "calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int) ## ASSEMBLE THE RESULTANT PARTICLE.. resultant_particle_data = [{DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.DCL_CONTROLLER_TIMESTAMP, DataParticleKey.VALUE: dcl_controller_timestamp}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.UNIQUE_ID, DataParticleKey.VALUE: unique_id_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_TYPE, DataParticleKey.VALUE: record_type_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_TIME, DataParticleKey.VALUE: record_time_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.CLOCK_ACTIVE, DataParticleKey.VALUE: clock_active}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORDING_ACTIVE, DataParticleKey.VALUE: recording_active}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_TIME, DataParticleKey.VALUE: record_end_on_time}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_MEMORY_FULL, DataParticleKey.VALUE: record_memory_full}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_ERROR, DataParticleKey.VALUE: record_end_on_error}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.DATA_DOWNLOAD_OK, DataParticleKey.VALUE: data_download_ok}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.FLASH_MEMORY_OPEN, DataParticleKey.VALUE: flash_memory_open}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_PRESTART, DataParticleKey.VALUE: battery_low_prestart}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_MEASUREMENT, DataParticleKey.VALUE: battery_low_measurement}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_BLANK, DataParticleKey.VALUE: battery_low_blank}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_EXTERNAL, DataParticleKey.VALUE: battery_low_external}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE1_FAULT, DataParticleKey.VALUE: external_device1_fault}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE2_FAULT, DataParticleKey.VALUE: external_device2_fault}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE3_FAULT, DataParticleKey.VALUE: external_device3_fault}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.FLASH_ERASED, DataParticleKey.VALUE: flash_erased}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.POWER_ON_INVALID, DataParticleKey.VALUE: power_on_invalid}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_DATA_RECORDS, DataParticleKey.VALUE: num_data_records_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_ERROR_RECORDS, DataParticleKey.VALUE: num_error_records_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.NUM_BYTES_STORED, DataParticleKey.VALUE: num_bytes_stored_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.VOLTAGE_BATTERY, DataParticleKey.VALUE: voltage_battery_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclMetadataDataParticleKey.PASSED_CHECKSUM, DataParticleKey.VALUE: checksum_final}] log.debug("PhsenAbcdefDclMetadataDataParticle._build_parsed_values(): resultant_particle_data= %s", resultant_particle_data) return resultant_particle_data
def _build_parsed_values(self): """ Extracts PHSEN ABCDEF DCL Instrument data from the raw_data tuple. @returns result a list of dictionaries of particle data """ ## extract the time from the raw_data touple dcl_controller_timestamp = self.raw_data[0] ## convert the time converted_time = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp) ## set the converted time to the particle internal timestamp self.set_internal_timestamp(converted_time) log.debug(" XX ## XX ## XX ## XX PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time) ## extract the working_record string from the raw data tuple working_record = self.raw_data[1] log.trace("PhsenAbcdefDclInstrumentDataParticle._generate_particle(): dcl_controller_timestamp= %s, " "working_record= %s", dcl_controller_timestamp, working_record) log.trace("PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "Data Length= %s, working_record Length= %s", int(working_record[3:5], 16), len(working_record)) ## ## Begin saving particle data ## unique_id_ascii_hex = working_record[1:3] ## convert 2 ascii (hex) chars to int unique_id_int = int(unique_id_ascii_hex, 16) record_type_ascii_hex = working_record[5:7] ## convert 2 ascii (hex) chars to int record_type_int = int(record_type_ascii_hex, 16) record_time_ascii_hex = working_record[7:15] ## convert 8 ascii (hex) chars to int record_time_int = int(record_time_ascii_hex, 16) thermistor_start_ascii_hex = working_record[15:19] ## convert 4 ascii (hex) chars to int thermistor_start_int = int(thermistor_start_ascii_hex, 16) reference_light_measurements_list_int = self._create_reference_light_measurements_array(working_record) light_measurements_list_int = self._create_light_measurements_array(working_record) voltage_battery_ascii_hex = working_record[455:459] ## convert 4 ascii (hex) chars to int voltage_battery_int = int(voltage_battery_ascii_hex, 16) thermistor_end_ascii_hex = working_record[459:463] ## convert 4 ascii (hex) chars to int thermistor_end_int = int(thermistor_end_ascii_hex, 16) passed_checksum_ascii_hex = working_record[463:465] ## convert 2 ascii (hex) chars to int passed_checksum_int = int(passed_checksum_ascii_hex, 16) calculated_checksum = _calculate_working_record_checksum(working_record) log.trace("### ### ###PhsenAbcdefDclInstrumentDataParticle._generate_particle(): " "calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int) ## Per IDD, if the calculated checksum does not match the checksum in the record, ## use a checksum of zero in the resultant particle if passed_checksum_int != calculated_checksum: checksum_final = 0 else: checksum_final = 1 ## ASSEMBLE THE RESULTANT PARTICLE.. resultant_particle_data = [{DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.DCL_CONTROLLER_TIMESTAMP, DataParticleKey.VALUE: dcl_controller_timestamp}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.UNIQUE_ID, DataParticleKey.VALUE: unique_id_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TYPE, DataParticleKey.VALUE: record_type_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TIME, DataParticleKey.VALUE: record_time_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_START, DataParticleKey.VALUE: thermistor_start_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.REFERENCE_LIGHT_MEASUREMENTS, DataParticleKey.VALUE: reference_light_measurements_list_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.LIGHT_MEASUREMENTS, DataParticleKey.VALUE: light_measurements_list_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.VOLTAGE_BATTERY, DataParticleKey.VALUE: voltage_battery_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_END, DataParticleKey.VALUE: thermistor_end_int}, {DataParticleKey.VALUE_ID: PhsenAbcdefDclInstrumentDataParticleKey.PASSED_CHECKSUM, DataParticleKey.VALUE: checksum_final}] log.debug("PhsenAbcdefDclInstrumentDataParticle._build_parsed_values(): resultant_particle_data= %s", resultant_particle_data) return resultant_particle_data