def test_simple(self): """ Read test data and pull out data particles Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # Note: since the recovered and telemetered parser and particles are common # to each other, testing one is sufficient, will be completely tested # in driver tests parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(20) log.debug("*** test_simple Num particles %s", len(particles)) # check the first particle, which should be the metadata particle (recovered) test_data = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') # # # check all the values against expected results. # for i in range(len(particles)): self.assert_result(test_data['data'][i], particles[i]) stream_handle.close()
def test_extra_data(self): """ Ensure that bad data is skipped when it exists. """ # the first 2nd and 8th data record in this file are corrupted by adding additional # data values separated by tabs and will be ignored # we expect to get the metadata particle and only the valid # engineering data particles file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt') stream_handle = open(file_path, 'rU') log.info(self.exception_callback_value) parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), stream_handle, self.exception_callback) particles = parser.get_records(18) self.assert_particles(particles, 'WC_HMR_extra_data_values.yml', RESOURCE_PATH) self.assertTrue(self.exception_callback_value is not None) self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException)) # expect to see a recoverable sample exception in the log log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value) self.assertTrue(len(particles) == 18) stream_handle.close()
def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # Note: since the recovered and telemetered parser and particles are common # to each other, testing one is sufficient, will be completely tested # in driver tests parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_TELEMETERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # try to get 2000 particles, there are more data records in the file # so should get 2000 including the meta data particles = parser.get_records(2000) log.debug("*** test_get_many Num particles %s", len(particles)) self.assertEqual(len(particles), 2000) stream_handle.close()
def test_simple(self): """ Read test data and pull out data particles Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # Note: since the recovered and telemetered parser and particles are common # to each other, testing one is sufficient, will be completely tested # in driver tests parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(20) log.debug("*** test_simple Num particles %s", len(particles)) # check the first particle, which should be the metadata particle (recovered) test_data = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') # # # check all the values against expected results. # for i in range(len(particles)): self.assert_result(test_data['data'][i], particles[i]) stream_handle.close()
def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # Note: since the recovered and telemetered parser and particles are common # to each other, testing one is sufficient, will be completely tested # in driver tests parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_TELEMETERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # try to get 2000 particles, there are more data records in the file # so should get 2000 including the meta data particles = parser.get_records(2000) log.debug("*** test_get_many Num particles %s", len(particles)) self.assertEqual(len(particles), 2000) stream_handle.close()
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ # the first and 7th data record in this file are corrupted and will be ignored # we expect to get the metadata particle with the # timestamp from the 2nd data record and all of the valid engineering # data records file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt') stream_handle = open(file_path, 'r') log.info(self.exception_callback_value) parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # 18 particles particles = parser.get_records(18) expected_results = self.get_dict_from_yml('WC_HMR_bad_data_records.yml') self.assertTrue(len(particles) == 18) for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException)) stream_handle.close()
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ # the first and 7th data record in this file are corrupted and will be ignored # we expect to get the metadata particle with the # timestamp from the 2nd data record and all of the valid engineering # data records file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt') stream_handle = open(file_path, 'rU') log.info(self.exception_callback_value) parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), stream_handle, self.exception_callback) # 18 particles particles = parser.get_records(18) self.assert_particles(particles, 'WC_HMR_bad_data_records.yml', RESOURCE_PATH) self.assert_(isinstance(self.exception_callback_value[0], RecoverableSampleException)) stream_handle.close()
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ # the first and 7th data record in this file are corrupted and will be ignored # we expect to get the metadata particle with the # timestamp from the 2nd data record and all of the valid engineering # data records file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt') stream_handle = open(file_path, 'rU') log.info(self.exception_callback_value) parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), stream_handle, self.exception_callback) # 18 particles particles = parser.get_records(18) self.assert_particles(particles, 'WC_HMR_bad_data_records.yml', RESOURCE_PATH) self.assert_( isinstance(self.exception_callback_value[0], RecoverableSampleException)) stream_handle.close()
def test_extra_data(self): """ Ensure that bad data is skipped when it exists. """ # the first 2nd and 8th data record in this file are corrupted by adding additional # data values separated by tabs and will be ignored # we expect to get the metadata particle and only the valid # engineering data particles file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt') stream_handle = open(file_path, 'r') log.info(self.exception_callback_value) parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(18) self.assertTrue(self.exception_callback_value is not None) self.assert_( isinstance(self.exception_callback_value, RecoverableSampleException)) # expect to see a recoverable sample exception in the log log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value) expected_results = self.get_dict_from_yml( 'WC_HMR_extra_data_values.yml') self.assertTrue(len(particles) == 18) # since the first two records were corrupted the first records received # should be metadata particle with the timestamp of the 3rd data row # and the engineering particle from the 3rd row for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) self.assert_( isinstance(self.exception_callback_value, RecoverableSampleException)) stream_handle.close()
def test_mid_state_start(self): """ This test makes sure that we retrieve the correct particles upon starting with an offset state. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # position 547 is the end of the 7th data record, which would have produced the # metadata particle and the first 7 engineering particles initial_state = { StateKey.POSITION: 549, StateKey.METADATA_EXTRACTED: True } parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) #expect to get the 8th and 9th engineering particles next particles = parser.get_records(2) log.debug("Num particles: %s", len(particles)) self.assertTrue(len(particles) == 2) expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') # skip the first 8 particles in the yml file due to mid state start offset = 8 for i in range(len(particles)): self.assert_result(expected_results['data'][i + offset], particles[i]) # now expect the state to be the end of the 9th data record and metadata sent the_new_state = { StateKey.POSITION: 627, StateKey.METADATA_EXTRACTED: True } log.debug("********** expected state: %s", the_new_state) log.debug("******** new parser state: %s", parser._state) self.assertTrue(parser._state == the_new_state) stream_handle.close()
def test_extra_data(self): """ Ensure that bad data is skipped when it exists. """ # the first 2nd and 8th data record in this file are corrupted by adding additional # data values separated by tabs and will be ignored # we expect to get the metadata particle and only the valid # engineering data particles file_path = os.path.join(RESOURCE_PATH, '11079364_EXTRA_DATA_WC_HMR.txt') stream_handle = open(file_path, 'r') log.info(self.exception_callback_value) parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(18) self.assertTrue(self.exception_callback_value is not None) self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException)) # expect to see a recoverable sample exception in the log log.debug('TEST EXTRA DATA exception call back is %s', self.exception_callback_value) expected_results = self.get_dict_from_yml('WC_HMR_extra_data_values.yml') self.assertTrue(len(particles) == 18) # since the first two records were corrupted the first records received # should be metadata particle with the timestamp of the 3rd data row # and the engineering particle from the 3rd row for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) self.assert_(isinstance(self.exception_callback_value, RecoverableSampleException)) stream_handle.close()
def create_yml(self): """ This utility creates a yml file Be sure to verify the results by eye before trusting! """ fid = open(os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt'), 'r') stream_handle = fid parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(20) self.particle_to_yml(particles, '11079364_WC_HMR_recov.yml') fid.close()
def create_yml(self): """ This utility creates a yml file Be sure to verify the results by eye before trusting! """ fid = open(os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt'), 'r') stream_handle = fid parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(20) self.particle_to_yml(particles, '11079364_WC_HMR_recov.yml') fid.close()
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # 11079419_PPB_OCR_20.yml has the metadata and the first 19 # engineering particles in it expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(2) log.debug("Num particles: %s", len(particles)) self.assertTrue(len(particles) == 2) for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) # position 945 is the byte at the start of the 18th data record new_state = {StateKey.POSITION: 945, StateKey.METADATA_EXTRACTED: True} parser.set_state(new_state) particles = parser.get_records(2) self.assertTrue(len(particles) == 2) # offset in the expected results offset = 18 for i in range(len(particles)): self.assert_result(expected_results['data'][i + offset], particles[i]) stream_handle.close()
def test_mid_state_start(self): """ This test makes sure that we retrieve the correct particles upon starting with an offset state. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # position 547 is the end of the 7th data record, which would have produced the # metadata particle and the first 7 engineering particles initial_state = {StateKey.POSITION: 549, StateKey.METADATA_EXTRACTED: True} parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) #expect to get the 8th and 9th engineering particles next particles = parser.get_records(2) log.debug("Num particles: %s", len(particles)) self.assertTrue(len(particles) == 2) expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') # skip the first 8 particles in the yml file due to mid state start offset = 8 for i in range(len(particles)): self.assert_result(expected_results['data'][i + offset], particles[i]) # now expect the state to be the end of the 9th data record and metadata sent the_new_state = {StateKey.POSITION: 627, StateKey.METADATA_EXTRACTED: True} log.debug("********** expected state: %s", the_new_state) log.debug("******** new parser state: %s", parser._state) self.assertTrue(parser._state == the_new_state) stream_handle.close()
def test_simple(self): """ Read test data and pull out data particles Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'rU') # Note: since the recovered and telemetered parser and particles are common # to each other, testing one is sufficient, will be completely tested # in driver tests parser = WcHmrCsppParser(self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), stream_handle, self.exception_callback) particles = parser.get_records(20) log.debug("*** test_simple Num particles %s", len(particles)) self.assert_particles(particles, '11079364_WC_HMR_recov.yml', RESOURCE_PATH) stream_handle.close()
def _build_parser(self, stream_handle): parser_config = { DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { METADATA_PARTICLE_CLASS_KEY: WcHmrMetadataRecoveredDataParticle, DATA_PARTICLE_CLASS_KEY: WcHmrEngRecoveredDataParticle, } } parser = WcHmrCsppParser(parser_config, stream_handle, self._exception_callback) return parser
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ # the first and 7th data record in this file are corrupted and will be ignored # we expect to get the metadata particle with the # timestamp from the 2nd data record and all of the valid engineering # data records file_path = os.path.join(RESOURCE_PATH, '11079364_BAD_WC_HMR.txt') stream_handle = open(file_path, 'r') log.info(self.exception_callback_value) parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # 18 particles particles = parser.get_records(18) expected_results = self.get_dict_from_yml( 'WC_HMR_bad_data_records.yml') self.assertTrue(len(particles) == 18) for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) self.assert_( isinstance(self.exception_callback_value, RecoverableSampleException)) stream_handle.close()
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, '11079364_WC_HMR.txt') stream_handle = open(file_path, 'r') # 11079419_PPB_OCR_20.yml has the metadata and the first 19 # engineering particles in it expected_results = self.get_dict_from_yml('11079364_WC_HMR_recov.yml') parser = WcHmrCsppParser( self.config.get(WcHmrDataTypeKey.WC_HMR_CSPP_RECOVERED), None, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) particles = parser.get_records(2) log.debug("Num particles: %s", len(particles)) self.assertTrue(len(particles) == 2) for i in range(len(particles)): self.assert_result(expected_results['data'][i], particles[i]) # position 945 is the byte at the start of the 18th data record new_state = {StateKey.POSITION: 945, StateKey.METADATA_EXTRACTED: True} parser.set_state(new_state) particles = parser.get_records(2) self.assertTrue(len(particles) == 2) # offset in the expected results offset = 18 for i in range(len(particles)): self.assert_result(expected_results['data'][i + offset], particles[i]) stream_handle.close()