def test_with_file_telem(self): with open(os.path.join(RESOURCE_PATH, 'first.rte.log'), 'rU') as file_handle: parser = RteODclParser(self.config, file_handle, self.exception_callback) particles = parser.get_records(5) self.assertEqual(len(particles), 2) self.assertListEqual(self.exception_callback_value, []) self.assert_particles(particles, 'first_rte.result.yml', RESOURCE_PATH)
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ with open(os.path.join(RESOURCE_PATH, 'bad_data.rte.log'), 'rU') as file_handle: parser = RteODclParser(self.config, file_handle, self.exception_callback) particles = parser.get_records(5) self.assertEqual(len(particles), 4) self.assertEqual(len(self.exception_callback_value), 1) self.assertIsInstance(self.exception_callback_value[0], RecoverableSampleException)
def test_bug_9692(self): """ Test to verify change made works with DCL timestamps containing seconds >59 """ with open(os.path.join(RESOURCE_PATH, '20131115A.rte.log'), 'rU') as file_handle: parser = RteODclParser(self.config, file_handle, self.exception_callback) result = parser.get_records(10) self.assertEqual(len(result), 5) self.assertListEqual(self.exception_callback_value, [])
def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ new_state = {StateKey.POSITION: 549} self.stream_handle = StringIO(RteODclParserUnitTestCase.TEST_DATA) self.parser = RteODclParser(self.config, new_state, self.stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, 703, self.timestamp3, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 857, self.timestamp4, self.particle_d)
def test_with_file_recov(self): config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.rte_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: 'RteODclParserRecoveredDataParticle' } with open(os.path.join(RESOURCE_PATH, 'four_samp.rte.log'), 'rU') as file_handle: parser = RteODclParser(config, file_handle, self.exception_callback) particles = parser.get_records(5) self.assertEqual(len(particles), 4) self.assertListEqual(self.exception_callback_value, []) self.assert_particles(particles, 'four_samp_rte_recov.result.yml', RESOURCE_PATH)
def test_bad_data(self): """ Ensure that bad data is skipped when it exists. """ self.stream_handle = StringIO(RteODclParserUnitTestCase.BAD_TEST_DATA) self.parser = RteODclParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, 395, self.timestamp1, self.particle_a) result = self.parser.get_records(1) self.assert_result(result, 703, self.timestamp3, self.particle_c)
def process(self): log = get_logger() with open(self._source_file_path, "rb") as file_handle: def exception_callback(exception): log.debug("Exception: %s", exception) self._particle_data_handler.setParticleDataCaptureFailure() parser = RteODclParser(self._parser_config, file_handle, exception_callback) driver = DataSetDriver(parser, self._particle_data_handler) driver.processFileStream() return self._particle_data_handler
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ new_state = {StateKey.POSITION: 549} self.stream_handle = StringIO(RteODclParserUnitTestCase.TEST_DATA) self.parser = RteODclParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, 395, self.timestamp1, self.particle_a) self.parser.set_state(new_state) result = self.parser.get_records(1) self.assert_result(result, 703, self.timestamp3, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 857, self.timestamp4, self.particle_d)
def process(self): log = get_logger() with open(self._sourceFilePath, "rb") as file_handle: def exception_callback(exception): log.debug("Exception: %s", exception) self._particleDataHdlrObj.setParticleDataCaptureFailure() parser = RteODclParser(self._parser_config, None, file_handle, lambda state, ingested: None, lambda data: None, exception_callback) driver = DataSetDriver(parser, self._particleDataHdlrObj) driver.processFileStream() return self._particleDataHdlrObj
def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ self.stream_handle = StringIO(RteODclParserUnitTestCase.TEST_DATA) self.parser = RteODclParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) self.assertEqual(result, [ self.particle_a, self.particle_b, self.particle_c, self.particle_d ]) self.assertEqual(self.parser._state[StateKey.POSITION], 857) self.assertEqual(self.state_callback_value[StateKey.POSITION], 857) self.assertEqual(self.publish_callback_value[0], self.particle_a) self.assertEqual(self.publish_callback_value[1], self.particle_b) self.assertEqual(self.publish_callback_value[2], self.particle_c) self.assertEqual(self.publish_callback_value[3], self.particle_d)
def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ self.stream_handle = StringIO(RteODclParserUnitTestCase.TEST_DATA) self.parser = RteODclParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result(result, 395, self.timestamp1, self.particle_a) result = self.parser.get_records(1) self.assert_result(result, 549, self.timestamp2, self.particle_b) result = self.parser.get_records(1) self.assert_result(result, 703, self.timestamp3, self.particle_c) result = self.parser.get_records(1) self.assert_result(result, 857, self.timestamp4, self.particle_d)
def _build_parser(self, parser_state, stream_in, data_key, file_in): """ Build the parser based on which data_key is input. The file name is only needed for mopak, and it just not passed in to the other parser builders @param parser_state previous parser state to initialize parser with @param stream_in handle of the opened file to parse @param data_key harvester / parser key @param file_in file name """ # get the config for the correct parser instance config = self._parser_config.get(data_key) if config is None: log.warn( 'Parser config does not exist for key = %s. Not building parser', data_key) raise ConfigurationException if data_key == DataTypeKey.CG_STC_ENG_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserDataParticle' }) parser = CgStcEngStcParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.CG_STC_ENG_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_stc_eng_stc', DataSetDriverConfigKeys.PARTICLE_CLASS: 'CgStcEngStcParserRecoveredDataParticle' }) parser = CgStcEngStcParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.MOPAK_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.mopak_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: None, # particle_class configuration does nothing for multi-particle parsers # put the class names in specific config parameters so the parser can get them # use real classes as objects instead of strings to make it easier DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { MopakParticleClassType.ACCEL_PARTCICLE_CLASS: MopakODclAccelParserDataParticle, MopakParticleClassType.RATE_PARTICLE_CLASS: MopakODclRateParserDataParticle } }) parser = MopakODclParser( config, parser_state, stream_in, file_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.MOPAK_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.mopak_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: None, # particle_class configuration does nothing for multi-particle parsers # put the class names in specific config parameters so the parser can get them # use real classes as objects instead of strings to make it easier DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { MopakParticleClassType.ACCEL_PARTCICLE_CLASS: MopakODclAccelParserRecoveredDataParticle, MopakParticleClassType.RATE_PARTICLE_CLASS: MopakODclRateParserRecoveredDataParticle } }) parser = MopakODclParser( config, parser_state, stream_in, file_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.RTE_TELEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.rte_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: 'RteODclParserDataParticle' }) parser = RteODclParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) elif data_key == DataTypeKey.RTE_RECOV: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.rte_o_dcl', DataSetDriverConfigKeys.PARTICLE_CLASS: 'RteODclParserRecoveredDataParticle' }) parser = RteODclParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) else: log.warn('Invalid Data_Key %s. Not building parser', data_key) raise ConfigurationException return parser