def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ self.stream_handle = StringIO( Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(4) self.assertEqual(result, [ self.particle_a_eng, self.particle_b_eng, self.particle_c_eng, self.particle_d_eng ]) self.assertEqual(self.parser._state[StateKey.POSITION], 128) self.assertEqual(self.state_callback_value[StateKey.POSITION], 128) self.assertEqual(self.publish_callback_value[0], self.particle_a_eng) self.assertEqual(self.publish_callback_value[1], self.particle_b_eng) self.assertEqual(self.publish_callback_value[2], self.particle_c_eng) self.assertEqual(self.publish_callback_value[3], self.particle_d_eng) self.assertEqual(self.file_ingested, True)
def parse(unused, source_file_path, particle_data_handler): config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle' } def state_callback(state, ingested): pass def pub_callback(data): log.trace("Found data: %s", data) def exception_callback(exception): particle_data_handler.setParticleDataCaptureFailure() stream_handle = open(source_file_path, 'rb') try: parser = Parad_k_stc_imodemParser(config, None, stream_handle, state_callback, pub_callback, exception_callback) driver = DataSetDriver(parser, particle_data_handler) driver.processFileStream() finally: stream_handle.close() return particle_data_handler
def test_simple(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ self.stream_handle = StringIO( Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT ) #turn into a data stream to look like file ingestion self.parser = Parad_k_stc_imodemParser( self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # last one is the link to the data source # next get engineering records result = self.parser.get_records(1) self.assert_result(result, 50, self.particle_a_eng, False) result = self.parser.get_records(1) self.assert_result(result, 76, self.particle_b_eng, False) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng, True) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.parser._state[StateKey.POSITION], 128) self.assertEqual(self.state_callback_value[StateKey.POSITION], 128) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_d_eng)
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj): config.add_configuration( os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml')) log = get_logger() parser_config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle' } def exception_callback(exception): log.debug("ERROR: %r", exception) particleDataHdlrObj.setParticleDataCaptureFailure() with open(sourceFilePath, 'rb') as stream_handle: parser = Parad_k_stc_imodemParser(parser_config, None, stream_handle, lambda state, ingested: None, lambda data: None, exception_callback) driver = DataSetDriver(parser, particleDataHdlrObj) driver.processFileStream() return particleDataHdlrObj
def test_bad_flags(self): """ test that we don't parse any records when the flags are not what we expect """ with self.assertRaises(SampleException): self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_BAD_FLAGS) self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback)
def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ new_state = {StateKey.POSITION:76} self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = Parad_k_stc_imodemParser(self.config, new_state, self.stream_handle, self.state_callback, self.pub_callback) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng, True)
def test_bad_data(self): """ Ensure that missing data causes us to miss records TODO: This test should be improved if we come up with a more accurate regex for the data sample """ self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_BAD_ENG) self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # next get engineering records result = self.parser.get_records(4) if len(result) == 4: self.fail("We got 4 records, the bad data should only make 3")
def test_long_stream(self): """ Test a long stream of data """ self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA) self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback) result = self.parser.get_records(32) self.assertEqual(result[0], self.particle_a_eng) self.assertEqual(result[-1], self.particle_last_eng) self.assertEqual(self.parser._state[StateKey.POSITION], 856) self.assertEqual(self.state_callback_value[StateKey.POSITION], 856) self.assertEqual(self.publish_callback_value[-1], self.particle_last_eng)
def _build_parser(self, parser_state, infile, data_key=None): """ Build and return the specified parser as indicated by the data_key. """ # # If the key is PARAD_K_STC, build the telemetered parser. # if data_key == DataTypeKey.PARAD_K_STC: config = self._parser_config[data_key] config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle' }) log.debug("My Config: %s", config) parser = Parad_k_stc_imodemParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) # # If the key is PARAD_K_STC_RECOVERED, build the recovered parser. # elif data_key == DataTypeKey.PARAD_K_STC_RECOVERED: config = self._parser_config[data_key] config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemRecoveredDataParticle' }) log.debug("My Config: %s", config) parser = Parad_k_stc_imodemRecoveredParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) # # If the key is one that we're not expecting, don't build any parser. # else: raise ConfigurationException('Parser configuration incorrect %s', data_key) return parser
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ new_state = {StateKey.POSITION:76} self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # set the new state, the essentially skips engineering a and b self.parser.set_state(new_state) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng, True)
def parse(unused, source_file_path, particle_data_handler): parser_config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle' } def exception_callback(exception): log.debug("ERROR: %r", exception) particle_data_handler.setParticleDataCaptureFailure() with open(source_file_path, 'rb') as stream_handle: parser = Parad_k_stc_imodemParser(parser_config, None, stream_handle, lambda state, ingested: None, lambda data: None, exception_callback) driver = DataSetDriver(parser, particle_data_handler) driver.processFileStream() return particle_data_handler
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj): from mi.logging import config config.add_configuration( os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml')) from mi.core.log import get_logger log = get_logger() from mi.dataset.dataset_driver import DataSetDriver, ParticleDataHandler from mi.dataset.parser.parad_k_stc_imodem import \ Parad_k_stc_imodemParser from mi.dataset.dataset_parser import DataSetDriverConfigKeys config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem', DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle' } try: if particleDataHdlrObj is not None: pass except NameError: particleDataHdlrObj = ParticleDataHandler() try: if sourceFilePath is not None: pass except NameError: try: sourceFilePath = sys.argv[1] except IndexError: print "Need a source file path" sys.exit(1) def state_callback(state, ingested): pass def pub_callback(data): log.trace("Found data: %s", data) def exception_callback(exception): particleDataHdlrObj.setParticleDataCaptureFailure() stream_handle = open(sourceFilePath, 'rb') try: parser = Parad_k_stc_imodemParser(config, None, stream_handle, state_callback, pub_callback, exception_callback) driver = DataSetDriver(parser, particleDataHdlrObj) driver.processFileStream() finally: stream_handle.close() stream_handle = open(sourceFilePath, 'rb') return particleDataHdlrObj