def test_simple(self):
"""
Read test data and pull out data particles
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
# check the first particle, which should be the metadata particle (recovered)
test_data = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
#
# # check all the values against expected results.
#
for i in range(len(particles)):
self.assert_result(test_data['data'][i], particles[i])
stream_handle.close()
def test_extra_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first 2nd and 8th data record in this file are corrupted by adding additional
# data values separated by tabs and will be ignored
# we expect to get the metadata particle and only the valid
# engineering data particles
file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle,
self.exception_callback)
particles = parser.get_records(18)
self.assert_particles(particles, 'WC_HMR_extra_data_values.yml', RESOURCE_PATH)
self.assertTrue(self.exception_callback_value is not None)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
# expect to see a recoverable sample exception in the log
log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value)
self.assertTrue(len(particles) == 18)
stream_handle.close()
def test_extra_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first 2nd and 8th data record in this file are corrupted by adding additional
# data values separated by tabs and will be ignored
# we expect to get the metadata particle and only the valid
# engineering data particles
file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle,
self.exception_callback)
particles = parser.get_records(18)
self.assert_particles(particles, 'WC_HMR_extra_data_values.yml', RESOURCE_PATH)
self.assertTrue(self.exception_callback_value is not None)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
# expect to see a recoverable sample exception in the log
log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value)
self.assertTrue(len(particles) == 18)
stream_handle.close()
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_TELEMETERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# try to get 2000 particles, there are more data records in the file
# so should get 2000 including the meta data
particles = parser.get_records(2000)
log.debug("*** test_get_many Num particles %s", len(particles))
self.assertEqual(len(particles), 2000)
stream_handle.close()
def test_simple(self):
"""
Read test data and pull out data particles
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
# check the first particle, which should be the metadata particle (recovered)
test_data = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
#
# # check all the values against expected results.
#
for i in range(len(particles)):
self.assert_result(test_data['data'][i], particles[i])
stream_handle.close()
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_TELEMETERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# try to get 2000 particles, there are more data records in the file
# so should get 2000 including the meta data
particles = parser.get_records(2000)
log.debug("*** test_get_many Num particles %s", len(particles))
self.assertEqual(len(particles), 2000)
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first and 7th data record in this file are corrupted and will be ignored
# we expect to get the metadata particle with the
# timestamp from the 2nd data record and all of the valid engineering
# data records
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt')
stream_handle = open(file_path, 'r')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
# 18 particles
particles = parser.get_records(18)
expected_results = self.get_dict_from_yml('WC_HMR_bad_data_records.yml')
self.assertTrue(len(particles) == 18)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException))
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first and 7th data record in this file are corrupted and will be ignored
# we expect to get the metadata particle with the
# timestamp from the 2nd data record and all of the valid engineering
# data records
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle,
self.exception_callback)
# 18 particles
particles = parser.get_records(18)
self.assert_particles(particles, 'WC_HMR_bad_data_records.yml', RESOURCE_PATH)
self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException))
stream_handle.close()
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first and 7th data record in this file are corrupted and will be ignored
# we expect to get the metadata particle with the
# timestamp from the 2nd data record and all of the valid engineering
# data records
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle, self.exception_callback)
# 18 particles
particles = parser.get_records(18)
self.assert_particles(particles, 'WC_HMR_bad_data_records.yml',
RESOURCE_PATH)
self.assert_(
isinstance(self.exception_callback_value[0],
RecoverableSampleException))
stream_handle.close()
def test_extra_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first 2nd and 8th data record in this file are corrupted by adding additional
# data values separated by tabs and will be ignored
# we expect to get the metadata particle and only the valid
# engineering data particles
file_path = os.path.join(RESOURCE_PATH,
'11079364_EXTRA_DATA_WC_HMR.txt')
stream_handle = open(file_path, 'r')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(18)
self.assertTrue(self.exception_callback_value is not None)
self.assert_(
isinstance(self.exception_callback_value,
RecoverableSampleException))
# expect to see a recoverable sample exception in the log
log.debug('TEST EXTRA DATA exception call back is %s',
self.exception_callback_value)
expected_results = self.get_dict_from_yml(
'WC_HMR_extra_data_values.yml')
self.assertTrue(len(particles) == 18)
# since the first two records were corrupted the first records received
# should be metadata particle with the timestamp of the 3rd data row
# and the engineering particle from the 3rd row
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
self.assert_(
isinstance(self.exception_callback_value,
RecoverableSampleException))
stream_handle.close()
def test_mid_state_start(self):
"""
This test makes sure that we retrieve the correct particles upon starting with an offset state.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# position 547 is the end of the 7th data record, which would have produced the
# metadata particle and the first 7 engineering particles
initial_state = {
StateKey.POSITION: 549,
StateKey.METADATA_EXTRACTED: True
}
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
initial_state, stream_handle, self.state_callback,
self.pub_callback, self.exception_callback)
#expect to get the 8th and 9th engineering particles next
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
# skip the first 8 particles in the yml file due to mid state start
offset = 8
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset],
particles[i])
# now expect the state to be the end of the 9th data record and metadata sent
the_new_state = {
StateKey.POSITION: 627,
StateKey.METADATA_EXTRACTED: True
}
log.debug("********** expected state: %s", the_new_state)
log.debug("******** new parser state: %s", parser._state)
self.assertTrue(parser._state == the_new_state)
stream_handle.close()
def test_extra_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first 2nd and 8th data record in this file are corrupted by adding additional
# data values separated by tabs and will be ignored
# we expect to get the metadata particle and only the valid
# engineering data particles
file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt')
stream_handle = open(file_path, 'r')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(18)
self.assertTrue(self.exception_callback_value is not None)
self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException))
# expect to see a recoverable sample exception in the log
log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value)
expected_results = self.get_dict_from_yml('WC_HMR_extra_data_values.yml')
self.assertTrue(len(particles) == 18)
# since the first two records were corrupted the first records received
# should be metadata particle with the timestamp of the 3rd data row
# and the engineering particle from the 3rd row
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException))
stream_handle.close()
def create_yml(self):
"""
This utility creates a yml file
Be sure to verify the results by eye before trusting!
"""
fid = open(os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt'), 'r')
stream_handle = fid
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
self.particle_to_yml(particles, '11079364_WC_HMR_recov.yml')
fid.close()
def create_yml(self):
"""
This utility creates a yml file
Be sure to verify the results by eye before trusting!
"""
fid = open(os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt'), 'r')
stream_handle = fid
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(20)
self.particle_to_yml(particles, '11079364_WC_HMR_recov.yml')
fid.close()
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# 11079419_PPB_OCR_20.yml has the metadata and the first 19
# engineering particles in it
expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
None, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# position 945 is the byte at the start of the 18th data record
new_state = {StateKey.POSITION: 945, StateKey.METADATA_EXTRACTED: True}
parser.set_state(new_state)
particles = parser.get_records(2)
self.assertTrue(len(particles) == 2)
# offset in the expected results
offset = 18
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset], particles[i])
stream_handle.close()
def test_mid_state_start(self):
"""
This test makes sure that we retrieve the correct particles upon starting with an offset state.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# position 547 is the end of the 7th data record, which would have produced the
# metadata particle and the first 7 engineering particles
initial_state = {StateKey.POSITION: 549, StateKey.METADATA_EXTRACTED: True}
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
initial_state, stream_handle,
self.state_callback, self.pub_callback,
self.exception_callback)
#expect to get the 8th and 9th engineering particles next
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
# skip the first 8 particles in the yml file due to mid state start
offset = 8
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset], particles[i])
# now expect the state to be the end of the 9th data record and metadata sent
the_new_state = {StateKey.POSITION: 627, StateKey.METADATA_EXTRACTED: True}
log.debug("********** expected state: %s", the_new_state)
log.debug("******** new parser state: %s", parser._state)
self.assertTrue(parser._state == the_new_state)
stream_handle.close()
def test_simple(self):
"""
Read test data and pull out data particles
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
self.assert_particles(particles, '11079364_WC_HMR_recov.yml', RESOURCE_PATH)
stream_handle.close()
def test_simple(self):
"""
Read test data and pull out data particles
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'rU')
# Note: since the recovered and telemetered parser and particles are common
# to each other, testing one is sufficient, will be completely tested
# in driver tests
parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED),
stream_handle,
self.exception_callback)
particles = parser.get_records(20)
log.debug("*** test_simple Num particles %s", len(particles))
self.assert_particles(particles, '11079364_WC_HMR_recov.yml', RESOURCE_PATH)
stream_handle.close()
def _build_parser(self, stream_handle):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
WcHmrMetadataRecoveredDataParticle,
DATA_PARTICLE_CLASS_KEY: WcHmrEngRecoveredDataParticle,
}
}
parser = WcHmrCsppParser(parser_config, stream_handle,
self._exception_callback)
return parser
def test_bad_data(self):
"""
Ensure that bad data is skipped when it exists.
"""
# the first and 7th data record in this file are corrupted and will be ignored
# we expect to get the metadata particle with the
# timestamp from the 2nd data record and all of the valid engineering
# data records
file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt')
stream_handle = open(file_path, 'r')
log.info(self.exception_callback_value)
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
# 18 particles
particles = parser.get_records(18)
expected_results = self.get_dict_from_yml(
'WC_HMR_bad_data_records.yml')
self.assertTrue(len(particles) == 18)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
self.assert_(
isinstance(self.exception_callback_value,
RecoverableSampleException))
stream_handle.close()
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt')
stream_handle = open(file_path, 'r')
# 11079419_PPB_OCR_20.yml has the metadata and the first 19
# engineering particles in it
expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml')
parser = WcHmrCsppParser(
self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None,
stream_handle, self.state_callback, self.pub_callback,
self.exception_callback)
particles = parser.get_records(2)
log.debug("Num particles: %s", len(particles))
self.assertTrue(len(particles) == 2)
for i in range(len(particles)):
self.assert_result(expected_results['data'][i], particles[i])
# position 945 is the byte at the start of the 18th data record
new_state = {StateKey.POSITION: 945, StateKey.METADATA_EXTRACTED: True}
parser.set_state(new_state)
particles = parser.get_records(2)
self.assertTrue(len(particles) == 2)
# offset in the expected results
offset = 18
for i in range(len(particles)):
self.assert_result(expected_results['data'][i + offset],
particles[i])
stream_handle.close()
