def _generate_internal_timestamp(record_dict):
"""
Generates the internal timestamp from the given DCL Controller Timestamp.
:param record_dict: dictionary containing the dcl controller timestamp str parameter
:return: the internal timestamp
"""
return float(dcl_controller_timestamp_to_ntp_time(
record_dict[Pco2wAbcDataParticleKey.DCL_CONTROLLER_TIMESTAMP]))
def parse_file(self):
"""
Parse the zplsc_c log file (averaged condensed data).
Read file line by line. Values are extracted from lines containing condensed ASCII data
@return: dictionary of data values with the particle names as keys or None
"""
# Loop over all lines in the data file and parse the data to generate particles
for number, line in enumerate(self._stream_handle, start=1):
# Check if this is the dcl status log
match = DCL_LOG_MATCHER.match(line)
if match is not None:
log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number,
match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument phase status log
match = PHASE_STATUS_MATCHER.match(line)
if match is not None:
log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number,
match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument condensed ASCII data
match = SENSOR_DATA_MATCHER.match(line)
if match is not None:
log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number,
match.groups())
# Extract the condensed ASCII data from this line
data_dict = self.parse_line(match)
if data_dict is None:
log.error('Erroneous data found in line %s: %s', number,
line)
continue
# Convert the DCL timestamp into the particle timestamp
time_stamp = dcl_controller_timestamp_to_ntp_time(
match.group('dcl_timestamp'))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(ZplscCInstrumentDataParticle,
None, data_dict, time_stamp)
if particle is not None:
log.trace('Parsed particle: %s' % particle.generate_dict())
self._record_buffer.append(particle)
continue
# Error, line did not match any expected regex
self._exception_callback(
RecoverableSampleException('Unknown data found in line %s:%s' %
(number, line)))
def _generate_internal_timestamp(record_dict):
"""
Generates the internal timestamp from the given DCL Controller Timestamp.
:param record_dict: dictionary containing the dcl controller timestamp str parameter
:return: the internal timestamp
"""
return float(
dcl_controller_timestamp_to_ntp_time(
record_dict[Pco2wAbcDataParticleKey.DCL_CONTROLLER_TIMESTAMP]))
def __init__(self, raw_data, instrument_particle_map, *args, **kwargs):
super(DclInstrumentDataParticle, self).__init__(raw_data, *args, **kwargs)
# The particle timestamp is the DCL Controller timestamp.
# Convert the DCL controller timestamp string to NTP time (in seconds and microseconds).
dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP]
elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp)
self.set_internal_timestamp(elapsed_seconds_useconds)
self.instrument_particle_map = instrument_particle_map
def _extract_instrument_ntp_timestamp(self, inst_match):
"""
This function will create a timestamp to be used as the internal
timestamp for the instrument particle is generated.
"""
# calculate the instrument particle internal timestamp
# from the DCL timestamp.
return dcl_controller_timestamp_to_ntp_time(inst_match.group(
InstrumentDataMatchGroups.INST_GROUP_DCL_TIMESTAMP))
def _extract_instrument_ntp_timestamp(self, inst_match):
"""
This function will create a timestamp to be used as the internal
timestamp for the instrument particle is generated.
"""
# calculate the instrument particle internal timestamp
# from the DCL timestamp.
return dcl_controller_timestamp_to_ntp_time(inst_match.group(
InstrumentDataMatchGroups.INST_GROUP_DCL_TIMESTAMP))
def parse_file(self):
"""
Parse the zplsc_c log file (averaged condensed data).
Read file line by line. Values are extracted from lines containing condensed ASCII data
@return: dictionary of data values with the particle names as keys or None
"""
# Loop over all lines in the data file and parse the data to generate particles
for number, line in enumerate(self._stream_handle, start=1):
# Check if this is the dcl status log
match = DCL_LOG_MATCHER.match(line)
if match is not None:
log.trace("MATCHED DCL_LOG_MATCHER: %s: %s", number, match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument phase status log
match = PHASE_STATUS_MATCHER.match(line)
if match is not None:
log.trace("MATCHED PHASE_STATUS_MATCHER: %s: %s", number, match.groups())
# No data to extract, move on to the next line
continue
# Check if this is the instrument condensed ASCII data
match = SENSOR_DATA_MATCHER.match(line)
if match is not None:
log.trace("MATCHED SENSOR_DATA_MATCHER: %s: %s", number, match.groups())
# Extract the condensed ASCII data from this line
data_dict = self.parse_line(match)
if data_dict is None:
log.error('Erroneous data found in line %s: %s', number, line)
continue
# Convert the DCL timestamp into the particle timestamp
time_stamp = dcl_controller_timestamp_to_ntp_time(match.group('dcl_timestamp'))
# Extract a particle and append it to the record buffer
particle = self._extract_sample(
ZplscCInstrumentDataParticle, None, data_dict, time_stamp)
if particle is not None:
log.trace('Parsed particle: %s' % particle.generate_dict())
self._record_buffer.append(particle)
continue
# Error, line did not match any expected regex
self._exception_callback(
RecoverableSampleException('Unknown data found in line %s:%s' % (number, line)))
def __init__(self,
raw_data,
port_timestamp=None,
internal_timestamp=None,
preferred_timestamp=DataParticleKey.PORT_TIMESTAMP,
quality_flag=DataParticleValue.OK,
new_sequence=None):
super(SpkirAbjDclInstrumentDataParticle,
self).__init__(raw_data, port_timestamp, internal_timestamp,
preferred_timestamp, quality_flag, new_sequence)
# The particle timestamp is the DCL Controller timestamp.
# Convert the DCL controller timestamp string to NTP time (in seconds and microseconds).
dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP]
elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(
dcl_controller_timestamp)
self.set_internal_timestamp(elapsed_seconds_useconds)
def __init__(self, raw_data,
port_timestamp=None,
internal_timestamp=None,
preferred_timestamp=DataParticleKey.PORT_TIMESTAMP,
quality_flag=DataParticleValue.OK,
new_sequence=None):
super(DostaAbcdjmDclInstrumentDataParticle, self).__init__(raw_data,
port_timestamp,
internal_timestamp,
preferred_timestamp,
quality_flag,
new_sequence)
# The particle timestamp is the DCL Controller timestamp.
# Convert the DCL controller timestamp string to NTP time (in seconds and microseconds).
dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP]
elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp)
self.set_internal_timestamp(elapsed_seconds_useconds)
def _build_parsed_values(self):
"""
Build parsed values for Recovered and Telemetered Instrument Data Particle.
"""
# Generate a particle by calling encode_value for each entry
# in the Instrument Particle Mapping table,
# where each entry is a tuple containing the particle field name,
# an index into the match groups (which is what has been stored in raw_data),
# and a function to use for data conversion.
# The particle timestamp is the DCL Controller timestamp.
# Convert the DCL controller timestamp string to NTP time (in seconds and microseconds).
dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP]
elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp)
self.set_internal_timestamp(elapsed_seconds_useconds)
return [self._encode_value(name, self.raw_data[group], function)
for name, group, function in INSTRUMENT_PARTICLE_MAP]
def _build_parsed_values(self):
"""
Build parsed values for Recovered and Telemetered Instrument Data Particle.
"""
# Generate a particle by calling encode_value for each entry
# in the Instrument Particle Mapping table,
# where each entry is a tuple containing the particle field name,
# an index into the match groups (which is what has been stored in raw_data),
# and a function to use for data conversion.
# The particle timestamp is the DCL Controller timestamp.
# Convert the DCL controller timestamp string to NTP time (in seconds and microseconds).
dcl_controller_timestamp = self.raw_data[SENSOR_GROUP_TIMESTAMP]
elapsed_seconds_useconds = dcl_controller_timestamp_to_ntp_time(
dcl_controller_timestamp)
self.set_internal_timestamp(elapsed_seconds_useconds)
return [
self._encode_value(name, self.raw_data[group], function)
for name, group, function in INSTRUMENT_PARTICLE_MAP
]
def parse_file(self):
"""
Parse through the file, pulling single lines and comparing to the established patterns,
generating particles for data lines
"""
# initialize data for modem particle
first_timestamp = None
date_timestamp = None
distance = None
dsp_bat = None
xmit_bat = None
# initialize raw_data for CsppEngDclEngDataParticle
self._eng_data = [None] * 10
for line in self._stream_handle:
data_match = RECORD_MATCHER.match(line)
if data_match is None:
message = 'got malformed line %s ' % line
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
continue
if data_match.group('mode') == 'sent':
continue # skip sent messages, go to next line
timestamp_str = data_match.group('timestamp')
message = data_match.group('message')
if first_timestamp is None:
first_timestamp = timestamp_str # save the first timestamp for the modem particle
# save off header information for modem particle
# modem particle created after processing entire file.
range_match = RANGE_MATCHER.match(message)
if range_match:
distance = range_match.group('range')
continue # go to next line
dsp_match = DSP_MATCHER.match(message)
if dsp_match:
dsp_bat = dsp_match.group('dsp_bat')
continue # go to next line
xmit_match = XMIT_MATCHER.match(message)
if xmit_match:
xmit_bat = xmit_match.group('dsp_bat')
continue # go to next line
# process NMEA sentences
nmea_match = NMEA_MATCHER.match(message)
if nmea_match:
sentence = nmea_match.group('sentence')
checksum = int(nmea_match.group('checksum'),
16) # Convert to integer
# Note: NMEA checksums typically do not include the $ at the
# beginning of the sentence but it appears Wetlabs implemented
# it that way.
comp_checksum = self.calc_checksum(sentence)
if comp_checksum == checksum:
fields = sentence.split(',')
command = fields[5]
count = fields[6]
sentence_params = NMEA_SENTENCE_MAP.get(command)
if sentence_params is None:
# skip NMEA sentences we are not looking for
log.debug('NMEA sentence skipped %s', line)
continue # go to next line
expected_count, particle_class = sentence_params
if int(count) != expected_count:
message = 'did not get expected number of fields on line %s' % line
log.warn(message)
self._exception_callback(
RecoverableSampleException(message))
continue # go to next line
if particle_class == CsppEngDclEngDataParticle:
if command == 'DATE':
date_timestamp = timestamp_str # save timestamp from the DATE record
self.process_date(fields[7:])
elif command == 'PFS':
self._eng_data[1:3] = fields[7:9]
elif command == 'PST':
self.process_start(fields[7:])
elif command == 'ENA':
self._eng_data[5:7] = fields[7:9]
elif command == 'WHE':
self.process_wave(fields[7:])
else:
# Create particle and add to buffer
timestamp = dcl_controller_timestamp_to_ntp_time(
timestamp_str)
data_particle = self._extract_sample(
particle_class, None, fields[7:], timestamp)
self._record_buffer.append(data_particle)
else:
message = 'checksum failed on line %s' % line
log.warn(message)
self._exception_callback(
RecoverableSampleException(message))
# end for loop
# only send modem particle if we have a timestamp
# and at least one parameter
if first_timestamp and (distance or dsp_bat or xmit_bat):
timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp)
data_particle = self._extract_sample(CsppEngDclModemParticle, None,
[distance, dsp_bat, xmit_bat],
timestamp)
self._record_buffer.append(data_particle)
if any(self._eng_data
): # Publish CsppEngDclEngDataParticle if we have any data
if date_timestamp: # preference is DATE timestamp
timestamp = dcl_controller_timestamp_to_ntp_time(
date_timestamp)
else:
timestamp = dcl_controller_timestamp_to_ntp_time(
first_timestamp)
data_particle = self._extract_sample(CsppEngDclEngDataParticle,
None, self._eng_data,
timestamp)
self._record_buffer.append(data_particle)
def _build_parsed_values(self):
"""
Extracts PHSEN ABCDEF DCL Metadata data from raw_data.
@returns result a list of dictionaries of particle data
"""
## extract the time from the raw_data tuple
dcl_controller_timestamp = self.raw_data[0]
## convert the time
converted_time = dcl_controller_timestamp_to_ntp_time(
dcl_controller_timestamp)
## set the converted time to the particle internal timestamp
self.set_internal_timestamp(converted_time)
log.trace(
" XX ## XX ## XX ## XX PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"dcl_controller_timestamp= %s, converted DCL Time= %s",
dcl_controller_timestamp, converted_time)
## extract the working_record string from the raw data tuple
working_record = self.raw_data[1]
log.trace(
"PhsenAbcdefDclMetadataDataParticle._generate_particle(): dcl_controller_timestamp= %s, "
"working_record= %s", dcl_controller_timestamp, working_record)
log.trace(
"PhsenAbcdefDclMetadataDataParticle._generate_particle(): Data Length= %s, working_record Length= %s",
int(working_record[3:5], 16), len(working_record))
## Per the IDD, voltage_battery data is optional and not guaranteed to be included in every CONTROL
## data record. Nominal size of a metadata string without the voltage_battery data is 39 (including the #).
## Voltage data adds 4 ascii characters to that, so raw_data greater than 41 contains voltage data,
## anything smaller does not.
if len(working_record) >= 41:
have_voltage_battery_data = True
else:
have_voltage_battery_data = False
log.trace(
"PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"raw_data len= %s and contains: %s, have_voltage_battery_data= %s ",
len(working_record), working_record, have_voltage_battery_data)
##
## Begin saving particle data
##
unique_id_ascii_hex = working_record[1:3]
## convert 2 ascii (hex) chars to int
unique_id_int = int(unique_id_ascii_hex, 16)
record_type_ascii_hex = working_record[5:7]
## convert 2 ascii (hex) chars to int
record_type_int = int(record_type_ascii_hex, 16)
record_time_ascii_hex = working_record[7:15]
## convert 8 ascii (hex) chars to int
record_time_int = int(record_time_ascii_hex, 16)
## FLAGS
flags_ascii_hex = working_record[15:19]
## convert 4 ascii (hex) chars to list of binary data
flags_ascii_int = int(flags_ascii_hex, 16)
binary_list = [(flags_ascii_int >> x) & 0x1 for x in range(16)]
# log.debug("PhsenAbcdefDclMetadataDataParticle._generate_particle(): binary_list= %s", binary_list)
clock_active = binary_list[0]
recording_active = binary_list[1]
record_end_on_time = binary_list[2]
record_memory_full = binary_list[3]
record_end_on_error = binary_list[4]
data_download_ok = binary_list[5]
flash_memory_open = binary_list[6]
battery_low_prestart = binary_list[7]
battery_low_measurement = binary_list[8]
battery_low_blank = binary_list[9]
battery_low_external = binary_list[10]
external_device1_fault = binary_list[11]
external_device2_fault = binary_list[12]
external_device3_fault = binary_list[13]
flash_erased = binary_list[14]
power_on_invalid = binary_list[15]
num_data_records_ascii_hex = working_record[19:25]
## convert 6 ascii (hex) chars to int
num_data_records_int = int(num_data_records_ascii_hex, 16)
num_error_records_ascii_hex = working_record[25:31]
## convert 6 ascii (hex) chars to int
num_error_records_int = int(num_error_records_ascii_hex, 16)
num_bytes_stored_ascii_hex = working_record[31:37]
## convert 6 ascii (hex) chars to int
num_bytes_stored_int = int(num_bytes_stored_ascii_hex, 16)
calculated_checksum = _calculate_working_record_checksum(
working_record)
## Record may not have voltage data...
if have_voltage_battery_data:
voltage_battery_ascii_hex = working_record[37:41]
## convert 4 ascii (hex) chars to int
voltage_battery_int = int(voltage_battery_ascii_hex, 16)
passed_checksum_ascii_hex = working_record[41:43]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
else:
voltage_battery_int = None
passed_checksum_ascii_hex = working_record[37:39]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
log.debug(
"### ### ###PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"calculated_checksum= %s, passed_checksum_int= %s",
calculated_checksum, passed_checksum_int)
## ASSEMBLE THE RESULTANT PARTICLE..
resultant_particle_data = [{
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.DCL_CONTROLLER_TIMESTAMP,
DataParticleKey.VALUE:
dcl_controller_timestamp
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.UNIQUE_ID,
DataParticleKey.VALUE:
unique_id_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_TYPE,
DataParticleKey.VALUE:
record_type_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_TIME,
DataParticleKey.VALUE:
record_time_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.CLOCK_ACTIVE,
DataParticleKey.VALUE:
clock_active
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORDING_ACTIVE,
DataParticleKey.VALUE:
recording_active
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_TIME,
DataParticleKey.VALUE:
record_end_on_time
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_MEMORY_FULL,
DataParticleKey.VALUE:
record_memory_full
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_ERROR,
DataParticleKey.VALUE:
record_end_on_error
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.DATA_DOWNLOAD_OK,
DataParticleKey.VALUE:
data_download_ok
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.FLASH_MEMORY_OPEN,
DataParticleKey.VALUE:
flash_memory_open
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_PRESTART,
DataParticleKey.VALUE:
battery_low_prestart
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_MEASUREMENT,
DataParticleKey.VALUE:
battery_low_measurement
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_BLANK,
DataParticleKey.VALUE:
battery_low_blank
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_EXTERNAL,
DataParticleKey.VALUE:
battery_low_external
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE1_FAULT,
DataParticleKey.VALUE:
external_device1_fault
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE2_FAULT,
DataParticleKey.VALUE:
external_device2_fault
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE3_FAULT,
DataParticleKey.VALUE:
external_device3_fault
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.FLASH_ERASED,
DataParticleKey.VALUE:
flash_erased
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.POWER_ON_INVALID,
DataParticleKey.VALUE:
power_on_invalid
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_DATA_RECORDS,
DataParticleKey.VALUE:
num_data_records_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_ERROR_RECORDS,
DataParticleKey.VALUE:
num_error_records_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_BYTES_STORED,
DataParticleKey.VALUE:
num_bytes_stored_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.VOLTAGE_BATTERY,
DataParticleKey.VALUE:
voltage_battery_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.PASSED_CHECKSUM,
DataParticleKey.VALUE:
checksum_final
}]
log.debug(
"PhsenAbcdefDclMetadataDataParticle._build_parsed_values(): resultant_particle_data= %s",
resultant_particle_data)
return resultant_particle_data
def _build_parsed_values(self):
"""
Extracts PHSEN ABCDEF DCL Instrument data from the raw_data tuple.
@returns result a list of dictionaries of particle data
"""
## extract the time from the raw_data touple
dcl_controller_timestamp = self.raw_data[0]
## convert the time
converted_time = dcl_controller_timestamp_to_ntp_time(
dcl_controller_timestamp)
## set the converted time to the particle internal timestamp
self.set_internal_timestamp(converted_time)
log.debug(
" XX ## XX ## XX ## XX PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"dcl_controller_timestamp= %s, converted DCL Time= %s",
dcl_controller_timestamp, converted_time)
## extract the working_record string from the raw data tuple
working_record = self.raw_data[1]
log.trace(
"PhsenAbcdefDclInstrumentDataParticle._generate_particle(): dcl_controller_timestamp= %s, "
"working_record= %s", dcl_controller_timestamp, working_record)
log.trace(
"PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"Data Length= %s, working_record Length= %s",
int(working_record[3:5], 16), len(working_record))
##
## Begin saving particle data
##
unique_id_ascii_hex = working_record[1:3]
## convert 2 ascii (hex) chars to int
unique_id_int = int(unique_id_ascii_hex, 16)
record_type_ascii_hex = working_record[5:7]
## convert 2 ascii (hex) chars to int
record_type_int = int(record_type_ascii_hex, 16)
record_time_ascii_hex = working_record[7:15]
## convert 8 ascii (hex) chars to int
record_time_int = int(record_time_ascii_hex, 16)
thermistor_start_ascii_hex = working_record[15:19]
## convert 4 ascii (hex) chars to int
thermistor_start_int = int(thermistor_start_ascii_hex, 16)
reference_light_measurements_list_int = self._create_reference_light_measurements_array(
working_record)
light_measurements_list_int = self._create_light_measurements_array(
working_record)
voltage_battery_ascii_hex = working_record[455:459]
## convert 4 ascii (hex) chars to int
voltage_battery_int = int(voltage_battery_ascii_hex, 16)
thermistor_end_ascii_hex = working_record[459:463]
## convert 4 ascii (hex) chars to int
thermistor_end_int = int(thermistor_end_ascii_hex, 16)
passed_checksum_ascii_hex = working_record[463:465]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
calculated_checksum = _calculate_working_record_checksum(
working_record)
log.trace(
"### ### ###PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"calculated_checksum= %s, passed_checksum_int= %s",
calculated_checksum, passed_checksum_int)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
## ASSEMBLE THE RESULTANT PARTICLE..
resultant_particle_data = [{
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.DCL_CONTROLLER_TIMESTAMP,
DataParticleKey.VALUE:
dcl_controller_timestamp
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.UNIQUE_ID,
DataParticleKey.VALUE:
unique_id_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TYPE,
DataParticleKey.VALUE:
record_type_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TIME,
DataParticleKey.VALUE:
record_time_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_START,
DataParticleKey.VALUE:
thermistor_start_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.
REFERENCE_LIGHT_MEASUREMENTS,
DataParticleKey.VALUE:
reference_light_measurements_list_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.LIGHT_MEASUREMENTS,
DataParticleKey.VALUE:
light_measurements_list_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.VOLTAGE_BATTERY,
DataParticleKey.VALUE:
voltage_battery_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_END,
DataParticleKey.VALUE:
thermistor_end_int
}, {
DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.PASSED_CHECKSUM,
DataParticleKey.VALUE:
checksum_final
}]
log.debug(
"PhsenAbcdefDclInstrumentDataParticle._build_parsed_values(): resultant_particle_data= %s",
resultant_particle_data)
return resultant_particle_data
def parse_file(self):
"""
Parse through the file, pulling single lines and comparing to the established patterns,
generating particles for data lines
"""
# initialize data for modem particle
first_timestamp = None
date_timestamp = None
distance = None
dsp_bat = None
xmit_bat = None
# initialize raw_data for CsppEngDclEngDataParticle
self._eng_data = [None] * 10
for line in self._stream_handle:
data_match = RECORD_MATCHER.match(line)
if data_match is None:
message = 'got malformed line %s ' % line
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
continue
if data_match.group('mode') == 'sent':
continue # skip sent messages, go to next line
timestamp_str = data_match.group('timestamp')
message = data_match.group('message')
if first_timestamp is None:
first_timestamp = timestamp_str # save the first timestamp for the modem particle
# save off header information for modem particle
# modem particle created after processing entire file.
range_match = RANGE_MATCHER.match(message)
if range_match:
distance = range_match.group('range')
continue # go to next line
dsp_match = DSP_MATCHER.match(message)
if dsp_match:
dsp_bat = dsp_match.group('dsp_bat')
continue # go to next line
xmit_match = XMIT_MATCHER.match(message)
if xmit_match:
xmit_bat = xmit_match.group('dsp_bat')
continue # go to next line
# process NMEA sentences
nmea_match = NMEA_MATCHER.match(message)
if nmea_match:
sentence = nmea_match.group('sentence')
checksum = int(nmea_match.group('checksum'), 16) # Convert to integer
# Note: NMEA checksums typically do not include the $ at the
# beginning of the sentence but it appears Wetlabs implemented
# it that way.
comp_checksum = self.calc_checksum(sentence)
if comp_checksum == checksum:
fields = sentence.split(',')
command = fields[5]
count = fields[6]
sentence_params = NMEA_SENTENCE_MAP.get(command)
if sentence_params is None:
# skip NMEA sentences we are not looking for
log.debug('NMEA sentence skipped %s', line)
continue # go to next line
expected_count, particle_class = sentence_params
if int(count) != expected_count:
message = 'did not get expected number of fields on line %s' % line
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
continue # go to next line
if particle_class == CsppEngDclEngDataParticle:
if command == 'DATE':
date_timestamp = timestamp_str # save timestamp from the DATE record
self.process_date(fields[7:])
elif command == 'PFS':
self._eng_data[1:3] = fields[7:9]
elif command == 'PST':
self.process_start(fields[7:])
elif command == 'ENA':
self._eng_data[5:7] = fields[7:9]
elif command == 'WHE':
self.process_wave(fields[7:])
else:
# Create particle and add to buffer
timestamp = dcl_controller_timestamp_to_ntp_time(timestamp_str)
data_particle = self._extract_sample(particle_class,
None,
fields[7:],
timestamp)
self._record_buffer.append(data_particle)
else:
message = 'checksum failed on line %s' % line
log.warn(message)
self._exception_callback(RecoverableSampleException(message))
# end for loop
# only send modem particle if we have a timestamp
# and at least one parameter
if first_timestamp and (distance or dsp_bat or xmit_bat):
timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp)
data_particle = self._extract_sample(CsppEngDclModemParticle,
None,
[distance, dsp_bat, xmit_bat],
timestamp)
self._record_buffer.append(data_particle)
if any(self._eng_data): # Publish CsppEngDclEngDataParticle if we have any data
if date_timestamp: # preference is DATE timestamp
timestamp = dcl_controller_timestamp_to_ntp_time(date_timestamp)
else:
timestamp = dcl_controller_timestamp_to_ntp_time(first_timestamp)
data_particle = self._extract_sample(CsppEngDclEngDataParticle,
None,
self._eng_data,
timestamp)
self._record_buffer.append(data_particle)
def _build_parsed_values(self):
"""
Extracts PHSEN ABCDEF DCL Metadata data from raw_data.
@returns result a list of dictionaries of particle data
"""
## extract the time from the raw_data tuple
dcl_controller_timestamp = self.raw_data[0]
## convert the time
converted_time = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp)
## set the converted time to the particle internal timestamp
self.set_internal_timestamp(converted_time)
log.trace(" XX ## XX ## XX ## XX PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time)
## extract the working_record string from the raw data tuple
working_record = self.raw_data[1]
log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): dcl_controller_timestamp= %s, "
"working_record= %s", dcl_controller_timestamp, working_record)
log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): Data Length= %s, working_record Length= %s",
int(working_record[3:5], 16), len(working_record))
## Per the IDD, voltage_battery data is optional and not guaranteed to be included in every CONTROL
## data record. Nominal size of a metadata string without the voltage_battery data is 39 (including the #).
## Voltage data adds 4 ascii characters to that, so raw_data greater than 41 contains voltage data,
## anything smaller does not.
if len(working_record) >= 41:
have_voltage_battery_data = True
else:
have_voltage_battery_data = False
log.trace("PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"raw_data len= %s and contains: %s, have_voltage_battery_data= %s ",
len(working_record), working_record, have_voltage_battery_data)
##
## Begin saving particle data
##
unique_id_ascii_hex = working_record[1:3]
## convert 2 ascii (hex) chars to int
unique_id_int = int(unique_id_ascii_hex, 16)
record_type_ascii_hex = working_record[5:7]
## convert 2 ascii (hex) chars to int
record_type_int = int(record_type_ascii_hex, 16)
record_time_ascii_hex = working_record[7:15]
## convert 8 ascii (hex) chars to int
record_time_int = int(record_time_ascii_hex, 16)
## FLAGS
flags_ascii_hex = working_record[15:19]
## convert 4 ascii (hex) chars to list of binary data
flags_ascii_int = int(flags_ascii_hex, 16)
binary_list = [(flags_ascii_int >> x) & 0x1 for x in range(16)]
# log.debug("PhsenAbcdefDclMetadataDataParticle._generate_particle(): binary_list= %s", binary_list)
clock_active = binary_list[0]
recording_active = binary_list[1]
record_end_on_time = binary_list[2]
record_memory_full = binary_list[3]
record_end_on_error = binary_list[4]
data_download_ok = binary_list[5]
flash_memory_open = binary_list[6]
battery_low_prestart = binary_list[7]
battery_low_measurement = binary_list[8]
battery_low_blank = binary_list[9]
battery_low_external = binary_list[10]
external_device1_fault = binary_list[11]
external_device2_fault = binary_list[12]
external_device3_fault = binary_list[13]
flash_erased = binary_list[14]
power_on_invalid = binary_list[15]
num_data_records_ascii_hex = working_record[19:25]
## convert 6 ascii (hex) chars to int
num_data_records_int = int(num_data_records_ascii_hex, 16)
num_error_records_ascii_hex = working_record[25:31]
## convert 6 ascii (hex) chars to int
num_error_records_int = int(num_error_records_ascii_hex, 16)
num_bytes_stored_ascii_hex = working_record[31:37]
## convert 6 ascii (hex) chars to int
num_bytes_stored_int = int(num_bytes_stored_ascii_hex, 16)
calculated_checksum = _calculate_working_record_checksum(working_record)
## Record may not have voltage data...
if have_voltage_battery_data:
voltage_battery_ascii_hex = working_record[37:41]
## convert 4 ascii (hex) chars to int
voltage_battery_int = int(voltage_battery_ascii_hex, 16)
passed_checksum_ascii_hex = working_record[41:43]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
else:
voltage_battery_int = None
passed_checksum_ascii_hex = working_record[37:39]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
log.debug("### ### ###PhsenAbcdefDclMetadataDataParticle._generate_particle(): "
"calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int)
## ASSEMBLE THE RESULTANT PARTICLE..
resultant_particle_data = [{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.DCL_CONTROLLER_TIMESTAMP,
DataParticleKey.VALUE: dcl_controller_timestamp},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.UNIQUE_ID,
DataParticleKey.VALUE: unique_id_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_TYPE,
DataParticleKey.VALUE: record_type_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_TIME,
DataParticleKey.VALUE: record_time_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.CLOCK_ACTIVE,
DataParticleKey.VALUE: clock_active},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORDING_ACTIVE,
DataParticleKey.VALUE: recording_active},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_TIME,
DataParticleKey.VALUE: record_end_on_time},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_MEMORY_FULL,
DataParticleKey.VALUE: record_memory_full},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.RECORD_END_ON_ERROR,
DataParticleKey.VALUE: record_end_on_error},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.DATA_DOWNLOAD_OK,
DataParticleKey.VALUE: data_download_ok},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.FLASH_MEMORY_OPEN,
DataParticleKey.VALUE: flash_memory_open},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_PRESTART,
DataParticleKey.VALUE: battery_low_prestart},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_MEASUREMENT,
DataParticleKey.VALUE: battery_low_measurement},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_BLANK,
DataParticleKey.VALUE: battery_low_blank},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.BATTERY_LOW_EXTERNAL,
DataParticleKey.VALUE: battery_low_external},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE1_FAULT,
DataParticleKey.VALUE: external_device1_fault},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE2_FAULT,
DataParticleKey.VALUE: external_device2_fault},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.EXTERNAL_DEVICE3_FAULT,
DataParticleKey.VALUE: external_device3_fault},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.FLASH_ERASED,
DataParticleKey.VALUE: flash_erased},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.POWER_ON_INVALID,
DataParticleKey.VALUE: power_on_invalid},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_DATA_RECORDS,
DataParticleKey.VALUE: num_data_records_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_ERROR_RECORDS,
DataParticleKey.VALUE: num_error_records_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.NUM_BYTES_STORED,
DataParticleKey.VALUE: num_bytes_stored_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.VOLTAGE_BATTERY,
DataParticleKey.VALUE: voltage_battery_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclMetadataDataParticleKey.PASSED_CHECKSUM,
DataParticleKey.VALUE: checksum_final}]
log.debug("PhsenAbcdefDclMetadataDataParticle._build_parsed_values(): resultant_particle_data= %s",
resultant_particle_data)
return resultant_particle_data
def _build_parsed_values(self):
"""
Extracts PHSEN ABCDEF DCL Instrument data from the raw_data tuple.
@returns result a list of dictionaries of particle data
"""
## extract the time from the raw_data touple
dcl_controller_timestamp = self.raw_data[0]
## convert the time
converted_time = dcl_controller_timestamp_to_ntp_time(dcl_controller_timestamp)
## set the converted time to the particle internal timestamp
self.set_internal_timestamp(converted_time)
log.debug(" XX ## XX ## XX ## XX PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"dcl_controller_timestamp= %s, converted DCL Time= %s", dcl_controller_timestamp, converted_time)
## extract the working_record string from the raw data tuple
working_record = self.raw_data[1]
log.trace("PhsenAbcdefDclInstrumentDataParticle._generate_particle(): dcl_controller_timestamp= %s, "
"working_record= %s", dcl_controller_timestamp, working_record)
log.trace("PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"Data Length= %s, working_record Length= %s", int(working_record[3:5], 16), len(working_record))
##
## Begin saving particle data
##
unique_id_ascii_hex = working_record[1:3]
## convert 2 ascii (hex) chars to int
unique_id_int = int(unique_id_ascii_hex, 16)
record_type_ascii_hex = working_record[5:7]
## convert 2 ascii (hex) chars to int
record_type_int = int(record_type_ascii_hex, 16)
record_time_ascii_hex = working_record[7:15]
## convert 8 ascii (hex) chars to int
record_time_int = int(record_time_ascii_hex, 16)
thermistor_start_ascii_hex = working_record[15:19]
## convert 4 ascii (hex) chars to int
thermistor_start_int = int(thermistor_start_ascii_hex, 16)
reference_light_measurements_list_int = self._create_reference_light_measurements_array(working_record)
light_measurements_list_int = self._create_light_measurements_array(working_record)
voltage_battery_ascii_hex = working_record[455:459]
## convert 4 ascii (hex) chars to int
voltage_battery_int = int(voltage_battery_ascii_hex, 16)
thermistor_end_ascii_hex = working_record[459:463]
## convert 4 ascii (hex) chars to int
thermistor_end_int = int(thermistor_end_ascii_hex, 16)
passed_checksum_ascii_hex = working_record[463:465]
## convert 2 ascii (hex) chars to int
passed_checksum_int = int(passed_checksum_ascii_hex, 16)
calculated_checksum = _calculate_working_record_checksum(working_record)
log.trace("### ### ###PhsenAbcdefDclInstrumentDataParticle._generate_particle(): "
"calculated_checksum= %s, passed_checksum_int= %s", calculated_checksum, passed_checksum_int)
## Per IDD, if the calculated checksum does not match the checksum in the record,
## use a checksum of zero in the resultant particle
if passed_checksum_int != calculated_checksum:
checksum_final = 0
else:
checksum_final = 1
## ASSEMBLE THE RESULTANT PARTICLE..
resultant_particle_data = [{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.DCL_CONTROLLER_TIMESTAMP,
DataParticleKey.VALUE: dcl_controller_timestamp},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.UNIQUE_ID,
DataParticleKey.VALUE: unique_id_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TYPE,
DataParticleKey.VALUE: record_type_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.RECORD_TIME,
DataParticleKey.VALUE: record_time_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_START,
DataParticleKey.VALUE: thermistor_start_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.REFERENCE_LIGHT_MEASUREMENTS,
DataParticleKey.VALUE: reference_light_measurements_list_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.LIGHT_MEASUREMENTS,
DataParticleKey.VALUE: light_measurements_list_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.VOLTAGE_BATTERY,
DataParticleKey.VALUE: voltage_battery_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.THERMISTOR_END,
DataParticleKey.VALUE: thermistor_end_int},
{DataParticleKey.VALUE_ID:
PhsenAbcdefDclInstrumentDataParticleKey.PASSED_CHECKSUM,
DataParticleKey.VALUE: checksum_final}]
log.debug("PhsenAbcdefDclInstrumentDataParticle._build_parsed_values(): resultant_particle_data= %s",
resultant_particle_data)
return resultant_particle_data
