def test_hardware_config_bad_checksum(self):
"""
The file used in this test has a bad checksum in the hardware configuration record.
Instrument metadata will still be produced but the fields from
the hardware config will NOT be included.
"""
log.debug('===== START TEST HARDWARE CONFIG BAD CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'bad_checksum_in_hardware_config_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'no_hardware_config_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST HARDWARE CONFIG BAD CHECKSUM =====')
def test_bad_diagnostic_checksum(self):
"""
The file used in this test has a record with a bad checksum.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
with open(
os.path.join(
RESOURCE_PATH,
'bad_diag_checksum_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'too_few_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
def test_missing_all_config(self):
"""
The file used in this test has no user configuration record.
Instrument metadata will still be produced but the fields from
the user config will NOT be included.
"""
log.debug('===== START TEST MISSING ALL CONFIG RECORDS =====')
with open(
os.path.join(RESOURCE_PATH,
'no_config_recs_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'no_config_recs_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST MISSING ALL CONFIG RECORDS =====')
def test_truncated_file(self):
"""
The file used in this test has a malformed (too short) record at
the end of the file.This results in 71 particles being retrieved
instead of 72.
"""
log.debug('===== START TEST FOUND TRUNCATED FILE =====')
with open(
os.path.join(RESOURCE_PATH,
'truncated_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'truncated_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND TRUNCATED FILE =====')
def test_invalid_record_id(self):
"""
The file used in this test has one record with an invalid ID byte.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST INVALID RECORD ID =====')
with open(
os.path.join(RESOURCE_PATH,
'bad_id_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'bad_id_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST INVALID RECORD ID =====')
def test_invalid_sync_byte(self):
"""
The file used in this test has extra bytes between records which need to be skipped
in order to process the correct number of particles. All records are still processed.
"""
log.debug('===== START TEST INVALID SYNC BYTE =====')
with open(
os.path.join(RESOURCE_PATH,
'extra_bytes_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST INVALID SYNC BYTE =====')
def test_random_diag_record(self):
"""
The file used in this test has a bad checksum in the head configuration record.
Instrument metadata will still be produced but the fields from
the head config will NOT be included.
"""
log.debug('===== START TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
with open(os.path.join(RESOURCE_PATH, 'random_diag_record_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'random_diag_record_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
def test_simple(self):
"""
Read files and verify that all expected particles can be read.
Verify that the contents of the particles are correct.
This is the happy path test.
"""
log.debug('===== START TEST SIMPLE =====')
# Test the telemetered version
with open(os.path.join(RESOURCE_PATH, 'VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST SIMPLE =====')
def test_jumbled(self):
"""
Read files and verify that all expected particles can be read.
This particular data file has the velocity data records
preceded by the diagnostics records, a situation not likely
to occur on a deployed instrument but anything is possible!
The logic in the parser will not produce an instrument metadata
particle (configuration data) until it encounters a velocity or
a diagnostics record. Assumes that all the configuration records are
at the beginning of the file. This is reasonable as the instrument is
configured before being deployed. So the config records would be stored
first. Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST SIMPLE NOT IN ORDER =====')
# Test the telemetered version
with open(os.path.join(RESOURCE_PATH, 'jumbled_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'jumbled_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST SIMPLE NOT IN ORDER =====')
def test_too_many_diagnostics_records(self):
"""
The file used in this test has 21 diagnostics records in the second set.
Twenty are expected. The records are all still processed.
The error is simply noted.
"""
log.debug('===== START TEST TOO MANY DIAGNOSTICS RECORDS =====')
with open(
os.path.join(RESOURCE_PATH,
'too_many_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 73
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'too_many_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST TOO MANY DIAGNOSTICS RECORDS =====')
def test_bad_diag_checksum_21_recs(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug('===== START TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')
with open(os.path.join(RESOURCE_PATH, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 118
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')
def test_bad_diag_checksum_21_recs(self):
"""
The file used in this test has a power record with a missing timestamp.
This results in 9 particles being retrieved instead of 10, and also result in the exception
callback being called.
"""
log.debug(
'===== START TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS ====='
)
with open(os.path.join(RESOURCE_PATH, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 118
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles,
'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug(
'===== END TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS ====='
)
def test_bad_diagnostic_checksum(self):
"""
The file used in this test has a record with a bad checksum.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'bad_diag_checksum_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'too_few_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST FOUND BAD DIAGNOSTICS CHECKSUM =====')
def test_invalid_record_id(self):
"""
The file used in this test has one record with an invalid ID byte.
This results in 71 particles being retrieved instead of 72.
"""
log.debug('===== START TEST INVALID RECORD ID =====')
with open(os.path.join(RESOURCE_PATH, 'bad_id_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'bad_id_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST INVALID RECORD ID =====')
def test_invalid_sync_byte(self):
"""
The file used in this test has extra bytes between records which need to be skipped
in order to process the correct number of particles. All records are still processed.
"""
log.debug('===== START TEST INVALID SYNC BYTE =====')
with open(os.path.join(RESOURCE_PATH, 'extra_bytes_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST INVALID SYNC BYTE =====')
def test_no_diag_recs(self):
"""
The file used in this test has a single diagnostic header record but no diagnostic
records. No diagnostic particles will be produced.
"""
log.debug('===== START TEST NO DIAGNOSTIC RECORDS FOUND =====')
with open(os.path.join(RESOURCE_PATH, 'no_diag_recs_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 51
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'no_diag_recs_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST NO DIAGNOSTIC RECORDS FOUND =====')
def test_missing_all_config(self):
"""
The file used in this test has no user configuration record.
Instrument metadata will still be produced but the fields from
the user config will NOT be included.
"""
log.debug('===== START TEST MISSING ALL CONFIG RECORDS =====')
with open(os.path.join(RESOURCE_PATH, 'no_config_recs_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'no_config_recs_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST MISSING ALL CONFIG RECORDS =====')
def test_truncated_file(self):
"""
The file used in this test has a malformed (too short) record at
the end of the file.This results in 71 particles being retrieved
instead of 72.
"""
log.debug('===== START TEST FOUND TRUNCATED FILE =====')
with open(os.path.join(RESOURCE_PATH, 'truncated_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 71
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'truncated_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST FOUND TRUNCATED FILE =====')
def test_too_many_diagnostics_records(self):
"""
The file used in this test has 21 diagnostics records in the second set.
Twenty are expected. The records are all still processed.
The error is simply noted.
"""
log.debug('===== START TEST TOO MANY DIAGNOSTICS RECORDS =====')
with open(os.path.join(RESOURCE_PATH, 'too_many_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 73
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'too_many_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST TOO MANY DIAGNOSTICS RECORDS =====')
def test_partial_configuration(self):
"""
Attempt to build a parser with a bad configuration.
"""
log.debug('===== START TEST PARTIAL CONFIGURATION =====')
with open(os.path.join(RESOURCE_PATH, 'VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
with self.assertRaises(ConfigurationException):
parser = VelptAbParser(self._incomplete_parser_config,
file_handle, self.exception_callback)
log.debug('===== END TEST PARTIAL CONFIGURATION =====')
def test_simple(self):
"""
Read files and verify that all expected particles can be read.
Verify that the contents of the particles are correct.
This is the happy path test.
"""
log.debug('===== START TEST SIMPLE =====')
# Test the telemetered version
with open(os.path.join(RESOURCE_PATH, 'VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST SIMPLE =====')
def test_random_diag_record(self):
"""
The file used in this test has a bad checksum in the head configuration record.
Instrument metadata will still be produced but the fields from
the head config will NOT be included.
"""
log.debug('===== START TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
with open(os.path.join(RESOURCE_PATH, 'random_diag_record_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'random_diag_record_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST RANDOM DIAGNOSTIC RECORD FOUND =====')
def test_hardware_config_bad_checksum(self):
"""
The file used in this test has a bad checksum in the hardware configuration record.
Instrument metadata will still be produced but the fields from
the hardware config will NOT be included.
"""
log.debug('===== START TEST HARDWARE CONFIG BAD CHECKSUM =====')
with open(os.path.join(RESOURCE_PATH, 'bad_checksum_in_hardware_config_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config,
file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles, 'no_hardware_config_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)
log.debug('===== END TEST HARDWARE CONFIG BAD CHECKSUM =====')
def test_no_diag_recs(self):
"""
The file used in this test has a single diagnostic header record but no diagnostic
records. No diagnostic particles will be produced.
"""
log.debug('===== START TEST NO DIAGNOSTIC RECORDS FOUND =====')
with open(os.path.join(RESOURCE_PATH, 'no_diag_recs_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
as file_handle:
num_particles_to_request = num_expected_particles = 51
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(
particles, 'no_diag_recs_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST NO DIAGNOSTIC RECORDS FOUND =====')
def test_jumbled(self):
"""
Read files and verify that all expected particles can be read.
This particular data file has the velocity data records
preceded by the diagnostics records, a situation not likely
to occur on a deployed instrument but anything is possible!
The logic in the parser will not produce an instrument metadata
particle (configuration data) until it encounters a velocity or
a diagnostics record. Assumes that all the configuration records are
at the beginning of the file. This is reasonable as the instrument is
configured before being deployed. So the config records would be stored
first. Verify that the contents of the particles are correct.
There should be no exceptions generated.
"""
log.debug('===== START TEST SIMPLE NOT IN ORDER =====')
# Test the telemetered version
with open(
os.path.join(RESOURCE_PATH,
'jumbled_VELPT_SN_11402_2014-07-02.aqd'),
'rb') as file_handle:
num_particles_to_request = num_expected_particles = 72
parser = VelptAbParser(self._parser_config, file_handle,
self.exception_callback)
particles = parser.get_records(num_particles_to_request)
self.assertEquals(len(particles), num_expected_particles)
self.assert_particles(particles,
'jumbled_VELPT_SN_11402_2014-07-02.yml',
RESOURCE_PATH)
log.debug('===== END TEST SIMPLE NOT IN ORDER =====')
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.velpt_ab_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
VelptAbParticleClassKey.METADATA_PARTICLE_CLASS:
VelptAbDiagnosticsHeaderParticle,
VelptAbParticleClassKey.DIAGNOSTICS_PARTICLE_CLASS:
VelptAbDiagnosticsDataParticle,
VelptAbParticleClassKey.INSTRUMENT_METADATA_PARTICLE_CLASS:
VelptAbInstrumentMetadataParticle,
VelptAbParticleClassKey.INSTRUMENT_PARTICLE_CLASS:
VelptAbInstrumentDataParticle
}
}
parser = VelptAbParser(parser_config, stream_handle,
self._exception_callback)
return parser
