def test_long_stream(self):
"""
Read test data and pull out multiple data particles
Assert that we have the correct number of particles
"""
log.debug('===== START TEST LONG STREAM =====')
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
# try to get 1044 particles, 1043 data records plus one metadata
particles = parser.get_records(1044)
log.debug("*** test_long_stream Num particles is: %s", len(particles))
self.assertEqual(len(particles), 1044)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.debug('===== END TEST LONG STREAM =====')
def test_no_header(self):
"""
Ensure that missing entire header will cause a RecoverableSampleException to be thrown
and the metadata particle will not be created
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_ACS_ACS_no_header.txt')
stream_handle = open(file_path, 'rb')
log.info(self.exception_callback_value)
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
parser.get_records(10)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(isinstance(self.exception_callback_value[i], RecoverableSampleException))
# bad records
self.assertEqual(self.count, 1)
stream_handle.close()
def test_no_trailing_tab(self):
"""
Ensure that we can handle records that do not have trailing tabs. If we encounter them,
no exception should be thrown and the particle should be created as usual.
The second and fourth data records do not have trailing tabs.
"""
log.debug('===== START TEST NO TRAILING TAB =====')
# This test file has some records with a trailing tab, and others do not
no_trailing_tab_file = '11079364_ACS_ACS_no_trailing_tab.txt'
file_path = os.path.join(RESOURCE_PATH, no_trailing_tab_file)
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(45)
log.debug("*** test_simple Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACS_ACS_no_trailing_tab.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.debug('===== END TEST NO TRAILING TAB =====')
def test_no_trailing_tab(self):
"""
Ensure that we can handle records that do not have trailing tabs. If we encounter them,
no exception should be thrown and the particle should be created as usual.
"""
# This test file has some records with a trailing tab, and others do not
no_trailing_tab_file = '11079364_ACS_ACS_no_trailing_tab.txt'
file_path = os.path.join(RESOURCE_PATH, no_trailing_tab_file)
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
# load a dictionary from the yml file
test_data = self.get_dict_from_yml('11079364_ACS_ACS_recov.yml')
# check all the values against expected results.
for i in range(len(particles)):
self.assert_result(test_data['data'][i], particles[i])
stream_handle.close()
def test_simple(self):
"""
Read test data and pull out the first particle (metadata).
Assert that the results are those we expected.
"""
log.debug('===== START TEST HAPPY PATH SINGLE =====')
file_path = os.path.join(RESOURCE_PATH, '11079364_ACS_ACS_one_data_record.txt')
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(1)
log.debug("*** test_simple Num particles %s", len(particles))
# check the values against expected results.
self.assert_particles(particles, '11079364_ACS_ACS_one_data_record.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.debug('===== END TEST HAPPY PATH SINGLE =====')
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
# load a dictionary from the yml file
test_data = self.get_dict_from_yml('11079364_ACS_ACS_recov.yml')
# check all the values against expected results.
for i in range(len(particles)):
self.assert_result(test_data['data'][i], particles[i])
stream_handle.close()
def test_long_stream(self):
"""
Read test data and pull out multiple data particles
Assert that we have the correct number of particles
"""
log.info('===== START TEST LONG STREAM =====')
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
# try to get 1044 particles, 1043 data records plus one metadata
particles = parser.get_records(1044)
log.info("*** test_long_stream Num particles is: %s", len(particles))
self.assertEqual(len(particles), 1044)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.info('===== END TEST LONG STREAM =====')
def test_real_file_2(self):
"""
Read recovered data from zip file attached to the IDD
and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.info('===== START TEST REAL FILE 2 =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, "11079419_ACS_ACS.txt")
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(845)
log.debug("*** test_real_file_2 Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079419_ACS_ACS_recov.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.info('===== END TEST REAL FILE 2 =====')
def test_real_file_2(self):
"""
Read recovered data from zip file attached to the IDD
and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.debug('===== START TEST REAL FILE 2 =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, "11079419_ACS_ACS.txt")
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(845)
log.debug("*** test_real_file_2 Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079419_ACS_ACS_recov.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.debug('===== END TEST REAL FILE 2 =====')
def test_simple(self):
"""
Read test data and pull out the first particle (metadata).
Assert that the results are those we expected.
"""
log.info('===== START TEST HAPPY PATH SINGLE =====')
file_path = os.path.join(RESOURCE_PATH,
'11079364_ACS_ACS_one_data_record.txt')
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(1)
log.debug("*** test_simple Num particles %s", len(particles))
# check the values against expected results.
self.assert_particles(particles,
'11079364_ACS_ACS_one_data_record.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.info('===== END TEST HAPPY PATH SINGLE =====')
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists and a RecoverableSampleException is thrown.
Note: every other data record has bad data (float instead of int, extra column etc.)
"""
# Data Record 1: timestamp is int
# Data Record 3: timestamp has non-digit
# Data Record 5: depth is int
# Data Record 7: suspect_timestamp is digit
# Data Record 9: serial number has letters
# Data Record 11: on seconds is missing
# Data Record 13: num wavelengths is letters
# Data Record 15: c ref dark is float
# Data Record 17: a c ref count is letters
# Data Record 19: begin with tab
# Data Record 21: begin with space
# Data Record 23: a sig count has a letter
# Data Record 25: external temp count is a float
# Data Record 27: internal temp count has a letter
# Data Record 29: pressure counts has a letter
# Data Record 31: has byte loss - from sample file in IDD
# Data Record 33: fields separated by space instead of tab
# Data Record 35: fields separated by multiple spaces instead of tab
# (between depth and Suspect Timestamp)
# Data Record 37: record ends with space then line feed
# Data Record 39: line starting with the timestamp, depth, and
# suspect timestamp, followed by all hex ascii chars
# Data Record 41: a line that is not data and does not start with Timestamp
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_ACS_ACS.txt')
stream_handle = open(file_path, 'rb')
log.info(self.exception_callback_value)
parser = OptaaDjCsppParser(
self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
parser.get_records(20)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(
isinstance(self.exception_callback_value[i],
RecoverableSampleException))
# bad records
self.assertEqual(self.count, 21)
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists and a RecoverableSampleException is thrown.
Note: every other data record has bad data (float instead of int, extra column etc.)
"""
# Data Record 1: timestamp is int
# Data Record 3: timestamp has non-digit
# Data Record 5: depth is int
# Data Record 7: suspect_timestamp is digit
# Data Record 9: serial number has letters
# Data Record 11: on seconds is missing
# Data Record 13: num wavelengths is letters
# Data Record 15: c ref dark is float
# Data Record 17: a c ref count is letters
# Data Record 19: begin with tab
# Data Record 21: begin with space
# Data Record 23: a sig count has a letter
# Data Record 25: external temp count is a float
# Data Record 27: internal temp count has a letter
# Data Record 29: pressure counts has a letter
# Data Record 31: has byte loss - from sample file in IDD
# Data Record 33: fields separated by space instead of tab
# Data Record 35: fields separated by multiple spaces instead of tab
# (between depth and Suspect Timestamp)
# Data Record 37: record ends with space then line feed
# Data Record 39: line starting with the timestamp, depth, and
# suspect timestamp, followed by all hex ascii chars
# Data Record 41: a line that is not data and does not start with Timestamp
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_ACS_ACS.txt')
stream_handle = open(file_path, 'rb')
log.info(self.exception_callback_value)
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
parser.get_records(20)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(isinstance(self.exception_callback_value[i], RecoverableSampleException))
# bad records
self.assertEqual(self.count, 21)
stream_handle.close()
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH, "11079419_ACS_ACS.txt"), O_MODE)
stream_handle = fid
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(1044)
self.particle_to_yml(particles, '11079419_ACS_ACS.yml')
fid.close()
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA), 'r')
stream_handle = fid
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
self.particle_to_yml(particles, '11079364_ACS_ACS_recov.yml')
fid.close()
def create_yml(self):
"""
This utility creates a yml file
"""
fid = open(os.path.join(RESOURCE_PATH,
"11079419_ACS_ACS.txt"),
O_MODE)
stream_handle = fid
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(1044)
self.particle_to_yml(particles, '11079419_ACS_ACS.yml')
fid.close()
def test_real_file(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.debug('===== START TEST REAL FILE =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(1044)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACS_ACS_recov.yml', RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
# Telemetered
file_path = os.path.join(RESOURCE_PATH, TELEMETERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._telemetered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACD_ACS_telem.yml', RESOURCE_PATH)
stream_handle.close()
log.debug('===== END TEST REAL FILE =====')
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
This test makes sure that we retrieve the correct particles upon starting with an offset state.
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'rb')
# position 2394 is the beginning of the second data record, which would have produced the
# metadata particle and the first instrument particle
initial_state = {
StateKey.POSITION: 2394,
StateKey.METADATA_EXTRACTED: True
}
parser = OptaaDjCsppParser(
self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
initial_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback)
# expect to get the 2nd and 3rd instrument particles next
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
expected_results = self.get_dict_from_yml('mid_state_start.yml')
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# now expect the state to be the beginning of 5th record (4th data record)
the_new_state = {
StateKey.POSITION: 6316,
StateKey.METADATA_EXTRACTED: True
}
log.debug("********** expected state: %s", the_new_state)
log.debug("******** new parser state: %s", parser._state)
self.assertTrue(parser._state == the_new_state)
stream_handle.close()
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'r')
# The yml file has the metadata and the first 19
# instrument particles in it
expected_results = self.get_dict_from_yml('11079364_ACS_ACS_recov.yml')
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# position 33765 is the byte at the start of the 18th data record
new_state = {StateKey.POSITION: 33765, StateKey.METADATA_EXTRACTED: True}
parser.set_state(new_state)
particles = parser.get_records(2)
self.assertTrue(len(particles) == 2)
# offset in the expected results, into the 18th result
offset = 18
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset], particles[i])
stream_handle.close()
def test_missing_source_file(self):
"""
Ensure that a missing source file line will cause a RecoverableSampleException to be thrown
and the metadata particle will not be created. However, an
instrument particle should still get created.
"""
log.debug('===== START TEST MISSING SOURCE FILE =====')
file_path = os.path.join(
RESOURCE_PATH, '11079364_ACS_ACS_missing_source_file_record.txt')
stream_handle = open(file_path, O_MODE)
log.debug(self.exception_callback_value)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(10)
log.debug("Exception callback value: %s",
self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(
isinstance(self.exception_callback_value[i],
RecoverableSampleException))
# 1 bad record
self.assertEqual(len(self.exception_callback_value), 1)
# check all the values against expected results.
self.assert_particles(
particles, '11079364_ACS_ACS_missing_source_file_record.yml',
RESOURCE_PATH)
stream_handle.close()
log.debug('===== END TEST MISSING SOURCE FILE =====')
def test_bad_state(self):
"""
Test that setting state to a "bad" state will cause DatasetParserException
to be raised
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'r')
# State is missing Position
state = {StateKey.METADATA_EXTRACTED: True}
with self.assertRaises(DatasetParserException):
OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# State is missing StateKey.METADATA_EXTRACTED
state = {StateKey.POSITION: 33765}
with self.assertRaises(DatasetParserException):
OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# State is a list instead of dict
state = [{StateKey.POSITION: 33765, StateKey.METADATA_EXTRACTED: True}]
with self.assertRaises(DatasetParserException):
OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
stream_handle.close()
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
This test makes sure that we retrieve the correct particles upon starting with an offset state.
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'rb')
# position 2394 is the beginning of the second data record, which would have produced the
# metadata particle and the first instrument particle
initial_state = {StateKey.POSITION: 2394, StateKey.METADATA_EXTRACTED: True}
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
initial_state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# expect to get the 2nd and 3rd instrument particles next
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
expected_results = self.get_dict_from_yml('mid_state_start.yml')
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# now expect the state to be the beginning of 5th record (4th data record)
the_new_state = {StateKey.POSITION: 6316, StateKey.METADATA_EXTRACTED: True}
log.debug("********** expected state: %s", the_new_state)
log.debug("******** new parser state: %s", parser._state)
self.assertTrue(parser._state == the_new_state)
stream_handle.close()
def test_no_header(self):
"""
Ensure that missing entire header will cause a RecoverableSampleException to be thrown
and the metadata particle will not be created. However, an
instrument particle should still get created.
"""
log.debug('===== START TEST NO HEADER =====')
file_path = os.path.join(RESOURCE_PATH, '11079364_ACS_ACS_no_header.txt')
stream_handle = open(file_path, O_MODE)
log.debug(self.exception_callback_value)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(10)
log.debug("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(isinstance(self.exception_callback_value[i], RecoverableSampleException))
# 1 bad record
self.assertEqual(len(self.exception_callback_value), 1)
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACS_ACS_no_header.yml', RESOURCE_PATH)
stream_handle.close()
log.debug('===== END TEST NO HEADER =====')
def test_no_trailing_tab(self):
"""
Ensure that we can handle records that do not have trailing tabs. If we encounter them,
no exception should be thrown and the particle should be created as usual.
The second and fourth data records do not have trailing tabs.
"""
log.info('===== START TEST NO TRAILING TAB =====')
# This test file has some records with a trailing tab, and others do not
no_trailing_tab_file = '11079364_ACS_ACS_no_trailing_tab.txt'
file_path = os.path.join(RESOURCE_PATH, no_trailing_tab_file)
stream_handle = open(file_path, O_MODE)
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(45)
log.debug("*** test_simple Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles,
'11079364_ACS_ACS_no_trailing_tab.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
log.info('===== END TEST NO TRAILING TAB =====')
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.optaa_dj_cspp',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: OptaaDjCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: OptaaDjCsppInstrumentRecoveredDataParticle
}
}
parser = OptaaDjCsppParser(parser_config,
stream_handle,
self._exception_callback)
return parser
def test_long_stream(self):
"""
Read test data and pull out multiple data particles
Assert that we have the correct number of particles
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = OptaaDjCsppParser(self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# try to get 1044 particles, 1043 data records plus one meta data
particles = parser.get_records(1044)
log.info("*** test_long_stream Num particles is: %s", len(particles))
self.assertEqual(len(particles), 1044)
stream_handle.close()
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
config = self._parser_config.get(data_key)
#
# If the key is RECOVERED, build the recovered parser.
#
if data_key == DataTypeKey.OPTAA_DJ_CSPP_RECOVERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: OptaaDjCsppMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: OptaaDjCsppInstrumentRecoveredDataParticle
}
})
#
# If the key is TELEMETERED, build the telemetered parser.
#
elif data_key == DataTypeKey.OPTAA_DJ_CSPP_TELEMETERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY: OptaaDjCsppMetadataTelemeteredDataParticle,
DATA_PARTICLE_CLASS_KEY: OptaaDjCsppInstrumentTelemeteredDataParticle
}
})
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException("Invalid data_key (%s) supplied to build parser" % data_key)
parser = OptaaDjCsppParser(
config,
parser_state,
infile,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback)
return parser
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, 'r')
# The yml file has the metadata and the first 19
# instrument particles in it
expected_results = self.get_dict_from_yml('11079364_ACS_ACS_recov.yml')
parser = OptaaDjCsppParser(
self.config.get(DataTypeKey.OPTAA_DJ_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# position 33765 is the byte at the start of the 18th data record
new_state = {
StateKey.POSITION: 33765,
StateKey.METADATA_EXTRACTED: True
}
parser.set_state(new_state)
particles = parser.get_records(2)
self.assertTrue(len(particles) == 2)
# offset in the expected results, into the 18th result
offset = 18
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset],
particles[i])
stream_handle.close()
def test_real_file(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
log.info('===== START TEST REAL FILE =====')
# Recovered
file_path = os.path.join(RESOURCE_PATH, RECOVERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(1044)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACS_ACS_recov.yml',
RESOURCE_PATH)
self.assertEquals(self.exception_callback_value, [])
stream_handle.close()
# Telemetered
file_path = os.path.join(RESOURCE_PATH, TELEMETERED_SAMPLE_DATA)
stream_handle = open(file_path, O_MODE)
parser = OptaaDjCsppParser(self._telemetered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_real_file Num particles %s", len(particles))
# check all the values against expected results.
self.assert_particles(particles, '11079364_ACD_ACS_telem.yml',
RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST REAL FILE =====')
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists and a RecoverableSampleException is thrown.
Note: every other data record has bad data (float instead of int, extra column etc.).
Data Record 1: timestamp is int
Data Record 3: timestamp has non-digit
Data Record 5: depth is int
Data Record 7: suspect_timestamp is digit
Data Record 9: serial number has letters
Data Record 11: on seconds is missing
Data Record 13: num wavelengths is letters
Data Record 15: c ref dark is float
Data Record 17: a c ref count is letters
Data Record 19: begin with tab
Data Record 21: begin with space
Data Record 23: a sig count has a letter
Data Record 25: external temp count is a float
Data Record 27: internal temp count has a letter
Data Record 29: pressure counts has a letter
Data Record 31: has byte loss - from sample file in IDD
Data Record 33: fields separated by space instead of tab
Data Record 35: fields separated by multiple spaces instead of tab
(between depth and Suspect Timestamp)
Data Record 37: record ends with space then line feed
Data Record 39: line starting with the timestamp, depth, and
suspect timestamp, followed by all hex ascii chars
Data Record 41: a line that is not data and does not start with Timestamp
Particles will still get created for the metadata and valid instrument records.
"""
log.debug('===== START TEST BAD DATA =====')
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_ACS_ACS.txt')
stream_handle = open(file_path, O_MODE)
log.debug(self.exception_callback_value)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle,
self.exception_callback)
particles = parser.get_records(25)
log.debug("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(isinstance(self.exception_callback_value[i], RecoverableSampleException))
# 21 bad records
self.assertEqual(len(self.exception_callback_value), 21)
# check all the values against expected results.
self.assert_particles(particles, '11079364_BAD_ACS_ACS.yml', RESOURCE_PATH)
stream_handle.close()
log.debug('===== END TEST BAD DATA =====')
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists and a RecoverableSampleException is thrown.
Note: every other data record has bad data (float instead of int, extra column etc.).
Data Record 1: timestamp is int
Data Record 3: timestamp has non-digit
Data Record 5: depth is int
Data Record 7: suspect_timestamp is digit
Data Record 9: serial number has letters
Data Record 11: on seconds is missing
Data Record 13: num wavelengths is letters
Data Record 15: c ref dark is float
Data Record 17: a c ref count is letters
Data Record 19: begin with tab
Data Record 21: begin with space
Data Record 23: a sig count has a letter
Data Record 25: external temp count is a float
Data Record 27: internal temp count has a letter
Data Record 29: pressure counts has a letter
Data Record 31: has byte loss - from sample file in IDD
Data Record 33: fields separated by space instead of tab
Data Record 35: fields separated by multiple spaces instead of tab
(between depth and Suspect Timestamp)
Data Record 37: record ends with space then line feed
Data Record 39: line starting with the timestamp, depth, and
suspect timestamp, followed by all hex ascii chars
Data Record 41: a line that is not data and does not start with Timestamp
Particles will still get created for the metadata and valid instrument records.
"""
log.info('===== START TEST BAD DATA =====')
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_ACS_ACS.txt')
stream_handle = open(file_path, O_MODE)
log.info(self.exception_callback_value)
parser = OptaaDjCsppParser(self._recovered_parser_config,
stream_handle, self.exception_callback)
particles = parser.get_records(25)
log.info("Exception callback value: %s", self.exception_callback_value)
self.assertTrue(self.exception_callback_value is not None)
for i in range(len(self.exception_callback_value)):
self.assert_(
isinstance(self.exception_callback_value[i],
RecoverableSampleException))
# 21 bad records
self.assertEqual(len(self.exception_callback_value), 21)
# check all the values against expected results.
self.assert_particles(particles, '11079364_BAD_ACS_ACS.yml',
RESOURCE_PATH)
stream_handle.close()
log.info('===== END TEST BAD DATA =====')
