def process_diagnostic_data(self):
"""
Handles the processing of diagnostic data particles and handles error processing if events
which should have occurred prior to receiving a diagnostic record did not happen.
"""
# As diagnostics records have the same format as velocity records
# you can use the same routine used to break down the velocity data
timestamp = VelptAbDataParticle.get_timestamp(self._current_record)
date_time_group = VelptAbDataParticle.get_date_time_string(
self._current_record)
self._diagnostics_data_dict = VelptAbDataParticle.generate_data_dict(
self._current_record)
# Check to see if the instrument metadata particle has been produced yet
# If not, produce it now as this is the first diagnostics record. This assumes
# that diagnostics data records will occur BEFORE velocity data records.
if not self._config_metadata_built:
self.build_instrument_metadata_particle(timestamp)
# Upon encountering the first diagnostics record, grab its timestamp
# for use in the diagnostics metadata particle. Produce that metadata
# particle now.
if self._first_diagnostics_record:
self._first_diagnostics_record = False
diagnostics_header_dict = VelptAbDataParticle.generate_diagnostics_header_dict(
date_time_group, self._diagnostics_header_record)
self._total_diagnostic_records = VelptAbDataParticle.\
get_diagnostics_count(self._diagnostics_header_record)
particle = self._extract_sample(self._metadata_class,
None,
diagnostics_header_dict,
internal_timestamp=timestamp)
self._diagnostic_header_published = True
self._record_buffer.append(particle)
# Cover the case where unexpected diagnostics records are encountered
elif ((not self._diagnostic_header_published) | (not self._sending_diagnostics))\
& (not self._bad_diagnostic_header):
self._total_diagnostic_records = self.DEFAULT_DIAGNOSTICS_COUNT
self._diagnostic_header_published = True
log.warning('Unexpected diagnostic data record encountered')
self._exception_callback(
RecoverableSampleException(
'Unexpected diagnostic data record encountered, not preceded by header'
))
particle = self._extract_sample(self._diagnostics_class,
None,
self._diagnostics_data_dict,
internal_timestamp=timestamp)
self._record_buffer.append(particle)
self._diagnostics_count += 1
def process_diagnostic_data(self):
"""
Handles the processing of diagnostic data particles and handles error processing if events
which should have occurred prior to receiving a diagnostic record did not happen.
"""
# As diagnostics records have the same format as velocity records
# you can use the same routine used to break down the velocity data
timestamp = VelptAbDataParticle.get_timestamp(self._current_record)
date_time_group = VelptAbDataParticle.get_date_time_string(self._current_record)
self._diagnostics_data_dict = VelptAbDataParticle.generate_data_dict(self._current_record)
# Check to see if the instrument metadata particle has been produced yet
# If not, produce it now as this is the first diagnostics record. This assumes
# that diagnostics data records will occur BEFORE velocity data records.
if not self._config_metadata_built:
self.build_instrument_metadata_particle(timestamp)
# Upon encountering the first diagnostics record, grab its timestamp
# for use in the diagnostics metadata particle. Produce that metadata
# particle now.
if self._first_diagnostics_record:
self._first_diagnostics_record = False
diagnostics_header_dict = VelptAbDataParticle.generate_diagnostics_header_dict(
date_time_group, self._diagnostics_header_record)
self._total_diagnostic_records = VelptAbDataParticle.\
get_diagnostics_count(self._diagnostics_header_record)
particle = self._extract_sample(self._metadata_class,
None,
diagnostics_header_dict,
internal_timestamp=timestamp)
self._diagnostic_header_published = True
self._record_buffer.append(particle)
# Cover the case where unexpected diagnostics records are encountered
elif ((not self._diagnostic_header_published) | (not self._sending_diagnostics))\
& (not self._bad_diagnostic_header):
self._total_diagnostic_records = self.DEFAULT_DIAGNOSTICS_COUNT
self._diagnostic_header_published = True
log.warning('Unexpected diagnostic data record encountered')
self._exception_callback(
RecoverableSampleException('Unexpected diagnostic data record encountered, not preceded by header'))
particle = self._extract_sample(self._diagnostics_class,
None,
self._diagnostics_data_dict,
internal_timestamp=timestamp)
self._record_buffer.append(particle)
self._diagnostics_count += 1
def process_velocity_data(self):
"""
Handles the processing of velocity data particles and handles error processing if events
which should have occurred prior to receiving a velocity record did not happen.
"""
# Get the timestamp of the velocity record in case we need it for the metadata particle.
timestamp = VelptAbDataParticle.get_timestamp(self._current_record)
# Check to see if the instrument metadata particle has been produced yet
# If not, produce it now as this is the first velocity record. This assumes
# that velocity data records will occur BEFORE diagnostics data records.
if not self._config_metadata_built:
self.build_instrument_metadata_particle(timestamp)
# If this flag is still indicating TRUE, it means we found NO diagnostic records.
# That's an error!
if self._first_diagnostics_record:
self._first_diagnostics_record = False
log.warning('No diagnostic records present, just a header.'
'No particles generated')
self._exception_callback(
RecoverableSampleException('No diagnostic records present, just a header.'
'No particles generated'))
# This flag indicates that diagnostics were being produced and now that
# the first velocity record has been encountered, it's time to match the
# number of diagnostics particles produced against the number of diagnostic
# records expected from the diagnostics header.
if self._sending_diagnostics:
self._sending_diagnostics = False
if self._total_diagnostic_records != self._diagnostics_count:
if self._diagnostics_count < self._total_diagnostic_records:
log.warning('Not enough diagnostics records, got %s, expected %s',
self._diagnostics_count, self._total_diagnostic_records)
self._exception_callback(
RecoverableSampleException('Not enough diagnostics records'))
elif self._diagnostics_count > self._total_diagnostic_records:
log.warning('Too many diagnostics records, got %s, expected %s',
self._diagnostics_count, self._total_diagnostic_records)
self._exception_callback(
RecoverableSampleException('Too many diagnostics records'))
self._diagnostics_count = 0
self._total_diagnostic_records = 0
velocity_data_dict = VelptAbDataParticle.generate_data_dict(self._current_record)
particle = self._extract_sample(self._velocity_data_class,
None,
velocity_data_dict,
internal_timestamp=timestamp)
self._record_buffer.append(particle)
def process_velocity_data(self):
"""
Handles the processing of velocity data particles and handles error processing if events
which should have occurred prior to receiving a velocity record did not happen.
"""
# Get the timestamp of the velocity record in case we need it for the metadata particle.
timestamp = VelptAbDataParticle.get_timestamp(self._current_record)
# Check to see if the instrument metadata particle has been produced yet
# If not, produce it now as this is the first velocity record. This assumes
# that velocity data records will occur BEFORE diagnostics data records.
if not self._config_metadata_built:
self.build_instrument_metadata_particle(timestamp)
# If this flag is still indicating TRUE, it means we found NO diagnostic records.
# That's an error!
if self._first_diagnostics_record:
self._first_diagnostics_record = False
log.warning('No diagnostic records present, just a header.'
'No particles generated')
self._exception_callback(
RecoverableSampleException('No diagnostic records present, just a header.'
'No particles generated'))
# This flag indicates that diagnostics were being produced and now that
# the first velocity record has been encountered, it's time to match the
# number of diagnostics particles produced against the number of diagnostic
# records expected from the diagnostics header.
if self._sending_diagnostics:
self._sending_diagnostics = False
if self._total_diagnostic_records != self._diagnostics_count:
if self._diagnostics_count < self._total_diagnostic_records:
log.warning('Not enough diagnostics records, got %s, expected %s',
self._diagnostics_count, self._total_diagnostic_records)
self._exception_callback(
RecoverableSampleException('Not enough diagnostics records'))
elif self._diagnostics_count > self._total_diagnostic_records:
log.warning('Too many diagnostics records, got %s, expected %s',
self._diagnostics_count, self._total_diagnostic_records)
self._exception_callback(
RecoverableSampleException('Too many diagnostics records'))
self._diagnostics_count = 0
self._total_diagnostic_records = 0
velocity_data_dict = VelptAbDataParticle.generate_data_dict(self._current_record)
particle = self._extract_sample(self._velocity_data_class,
None,
velocity_data_dict,
timestamp)
self._record_buffer.append(particle)
def build_instrument_metadata_particle(self, timestamp):
"""
The instrument metadata particle is built from three separate records:
the hardware configuration record, the head configuration record and the
user configuration record. These should all be concentrated at the very
beginning of the recovered data file. This assumption is made because the
instrument is configured before being deployed so the records holding
this data would be stored first. The data files seen as of the date this
code was written all start with the three config records, then a quantity
of velocity data records, then a group of diagnostics (header plus data
records) followed by more velocity data records. This sequence can be
repeated a number of times in one file, though the config data will only
occur once. It is remotely possible that a group of diagnostics could
occur before any velocity data records are encountered. Publishing of
the instrument metadata particle is triggered by encountering either the
first velocity data record or the first diagnostics data record.
Counting to see if all three configuration records were encountered and
then producing the instrument metadata particle was rejected as it is
remotely possible that one of the configuration records could be missing.
"""
self._config_metadata_built = True
date_time_group = VelptAbDataParticle.get_date_time_string(self._current_record)
instrument_metadata_dict = VelptAbDataParticle.generate_instrument_metadata_dict\
(date_time_group, self._hardware_config_dict, self._head_config_dict,
self._user_config_dict)
particle = self._extract_sample(self._instrument_metadata_class,
None,
instrument_metadata_dict,
internal_timestamp=timestamp)
self._record_buffer.append(particle)
# Check to see if all the configuration records were found
if not self._hardware_config_dict_generated:
# Log a warning for the missing hardware config record
log.warning(
'Hardware configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'Hardware configuration record invalid or not present in recovered data'))
if not self._head_config_dict_generated:
# Log a warning for the missing head config record
log.warning('Head configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'Head configuration record invalid or not present in recovered data'))
if not self._user_config_dict_generated:
# Log a warning for the missing user config record
log.warning('User configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'User configuration record invalid or not present in recovered data'))
def build_instrument_metadata_particle(self, timestamp):
"""
The instrument metadata particle is built from three separate records:
the hardware configuration record, the head configuration record and the
user configuration record. These should all be concentrated at the very
beginning of the recovered data file. This assumption is made because the
instrument is configured before being deployed so the records holding
this data would be stored first. The data files seen as of the date this
code was written all start with the three config records, then a quantity
of velocity data records, then a group of diagnostics (header plus data
records) followed by more velocity data records. This sequence can be
repeated a number of times in one file, though the config data will only
occur once. It is remotely possible that a group of diagnostics could
occur before any velocity data records are encountered. Publishing of
the instrument metadata particle is triggered by encountering either the
first velocity data record or the first diagnostics data record.
Counting to see if all three configuration records were encountered and
then producing the instrument metadata particle was rejected as it is
remotely possible that one of the configuration records could be missing.
"""
self._config_metadata_built = True
date_time_group = VelptAbDataParticle.get_date_time_string(self._current_record)
instrument_metadata_dict = VelptAbDataParticle.generate_instrument_metadata_dict\
(date_time_group, self._hardware_config_dict, self._head_config_dict,
self._user_config_dict)
particle = self._extract_sample(self._instrument_metadata_class,
None,
instrument_metadata_dict,
timestamp)
self._record_buffer.append(particle)
# Check to see if all the configuration records were found
if not self._hardware_config_dict_generated:
# Log a warning for the missing hardware config record
log.warning(
'Hardware configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'Hardware configuration record invalid or not present in recovered data'))
if not self._head_config_dict_generated:
# Log a warning for the missing head config record
log.warning('Head configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'Head configuration record invalid or not present in recovered data'))
if not self._user_config_dict_generated:
# Log a warning for the missing user config record
log.warning('User configuration record invalid or not present in recovered data')
self._exception_callback(RecoverableSampleException(
'User configuration record invalid or not present in recovered data'))
def __init__(self, config, file_handle, exception_callback):
self._record_buffer = []
self._calculated_checksum = 0
self._current_record = ''
self._velocity_data = False
self._diagnostic_header = False
self._diagnostic_header_published = False
self._diagnostic_data = False
self._build_hardware_config_data = False
self._build_head_config_data = False
self._build_user_config_data = False
self._end_of_file = False
self._sending_diagnostics = False
self._build_hardware_config_data = False
self._config_metadata_built = False
self._hardware_config_dict_generated = False
self._head_config_dict_generated = False
self._user_config_dict_generated = False
self._bad_diagnostic_header = False
self._build_config_metadata = False
self._first_diagnostics_record = False
self._diagnostics_count = 0
self._total_diagnostic_records = 0
self._velocity_data_dict = {}
self._diagnostics_header_dict = {}
self._diagnostics_data_dict = {}
self._hardware_config_dict = VelptAbDataParticle.generate_empty_hardware_config_dict(
)
self._head_config_dict = VelptAbDataParticle.generate_empty_head_config_dict(
)
self._user_config_dict = VelptAbDataParticle.generate_empty_user_config_dict(
)
self._instrument_metadata_dict = {}
self._diagnostics_header_record = ''
self._file_handle = file_handle
# Obtain the particle classes dictionary from the config data
if DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT in config:
particle_classes_dict = config.get(
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT)
# Set the metadata and data particle classes to be used later
if VelptAbParticleClassKey.METADATA_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.DIAGNOSTICS_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.INSTRUMENT_METADATA_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.INSTRUMENT_PARTICLE_CLASS in particle_classes_dict:
self._metadata_class = config[
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.METADATA_PARTICLE_CLASS]
self._diagnostics_class = config[
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.DIAGNOSTICS_PARTICLE_CLASS]
self._velocity_data_class = config[
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.INSTRUMENT_PARTICLE_CLASS]
self._instrument_metadata_class = config[
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.
INSTRUMENT_METADATA_PARTICLE_CLASS]
else:
log.error(
'Configuration missing metadata or data particle class key in particle classes dict'
)
raise ConfigurationException(
'Configuration missing metadata or data particle class key in particle classes dict'
)
else:
log.error('Configuration missing particle classes dict')
raise ConfigurationException(
'Configuration missing particle classes dict')
super(VelptAbParser, self).__init__(config, file_handle,
exception_callback)
def __init__(self,
config,
file_handle,
exception_callback):
self._record_buffer = []
self._calculated_checksum = 0
self._current_record = ''
self._velocity_data = False
self._diagnostic_header = False
self._diagnostic_header_published = False
self._diagnostic_data = False
self._build_hardware_config_data = False
self._build_head_config_data = False
self._build_user_config_data = False
self._end_of_file = False
self._sending_diagnostics = False
self._build_hardware_config_data = False
self._config_metadata_built = False
self._hardware_config_dict_generated = False
self._head_config_dict_generated = False
self._user_config_dict_generated = False
self._bad_diagnostic_header = False
self._build_config_metadata = False
self._first_diagnostics_record = False
self._diagnostics_count = 0
self._total_diagnostic_records = 0
self._velocity_data_dict = {}
self._diagnostics_header_dict = {}
self._diagnostics_data_dict = {}
self._hardware_config_dict = VelptAbDataParticle.generate_empty_hardware_config_dict()
self._head_config_dict = VelptAbDataParticle.generate_empty_head_config_dict()
self._user_config_dict = VelptAbDataParticle.generate_empty_user_config_dict()
self._instrument_metadata_dict = {}
self._diagnostics_header_record = ''
self._file_handle = file_handle
# Obtain the particle classes dictionary from the config data
if DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT in config:
particle_classes_dict = config.get(DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT)
# Set the metadata and data particle classes to be used later
if VelptAbParticleClassKey.METADATA_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.DIAGNOSTICS_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.INSTRUMENT_METADATA_PARTICLE_CLASS in particle_classes_dict and \
VelptAbParticleClassKey.INSTRUMENT_PARTICLE_CLASS in particle_classes_dict:
self._metadata_class = config[
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][VelptAbParticleClassKey.METADATA_PARTICLE_CLASS]
self._diagnostics_class = config[DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.DIAGNOSTICS_PARTICLE_CLASS]
self._velocity_data_class = config[DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.INSTRUMENT_PARTICLE_CLASS]
self._instrument_metadata_class = config[DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT][
VelptAbParticleClassKey.INSTRUMENT_METADATA_PARTICLE_CLASS]
else:
log.error(
'Configuration missing metadata or data particle class key in particle classes dict')
raise ConfigurationException(
'Configuration missing metadata or data particle class key in particle classes dict')
else:
log.error('Configuration missing particle classes dict')
raise ConfigurationException('Configuration missing particle classes dict')
super(VelptAbParser, self).__init__(config, file_handle, exception_callback)
