def _build_parser(self, parser_state, infile, data_key=None): """ Build and return the parser """ # Default the parser to None parser = None config = self._parser_config.get(data_key) # # If the key is WFP_ENG_STC_IMODEM_RECOVERED, build the Wfp_eng__stc_imodemParser parser and # provide a config that includes the specific recovered particle types. # if data_key == DataTypeKey.WFP_ENG_STC_IMODEM_RECOVERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'status_data_particle_class': WfpEngStcImodemStatusRecoveredDataParticle, 'start_data_particle_class': WfpEngStcImodemStartRecoveredDataParticle, 'engineering_data_particle_class': WfpEngStcImodemEngineeringRecoveredDataParticle } }) log.debug("My Config: %s", config) parser = WfpEngStcImodemParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state(state, data_key, ingested), self._data_callback, self._sample_exception_callback) # # If the key is WFP_ENG_STC_IMODEM_TELEMETERED, build the Wfp_eng__stc_imodemParser parser and # provide a config that includes the specific telemetered particle types. # elif data_key == DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'status_data_particle_class': WfpEngStcImodemStatusTelemeteredDataParticle, 'start_data_particle_class': WfpEngStcImodemStartTelemeteredDataParticle, 'engineering_data_particle_class': WfpEngStcImodemEngineeringTelemeteredDataParticle } }) log.debug("My Config: %s", config) parser = WfpEngStcImodemParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state(state, data_key, ingested), self._data_callback, self._sample_exception_callback) else: raise ConfigurationException return parser
def test_long_stream_telemetered(self): """ Test a long stream of data """ self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(1) self.assert_result(result, 24, self.particle_a_start_time_telem, False) result = self.parser.get_records(32) self.assertEqual(result[0], self.particle_a_eng_telem) self.assertEqual(result[-1], self.particle_last_eng_telem) self.assertEqual(self.parser._state[StateKey.POSITION], 856) self.assertEqual(self.state_callback_value[StateKey.POSITION], 856) self.assertEqual(self.publish_callback_value[-1], self.particle_last_eng_telem) result = self.parser.get_records(1) self.assert_result(result, 872, self.particle_a_status_telem, True)
def test_simple_telemetered(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(1) self.assert_result(result, 24, self.particle_a_start_time_telem, False) # next get engineering records result = self.parser.get_records(1) self.assert_result(result, 50, self.particle_a_eng_telem, False) result = self.parser.get_records(1) self.assert_result(result, 76, self.particle_b_eng_telem, False) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng_telem, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng_telem, True) # no data left, dont move the position result = self.parser.get_records(1) self.assertEqual(result, []) self.assertEqual(self.parser._state[StateKey.POSITION], 128) self.assertEqual(self.state_callback_value[StateKey.POSITION], 128) self.assert_(isinstance(self.publish_callback_value, list)) self.assertEqual(self.publish_callback_value[0], self.particle_d_eng_telem)
def test_get_many_telemetered(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(1) self.assert_result(result, 24, self.particle_a_start_time_telem, False) result = self.parser.get_records(4) self.assertEqual(result, [ self.particle_a_eng_telem, self.particle_b_eng_telem, self.particle_c_eng_telem, self.particle_d_eng_telem ]) self.assertEqual(self.parser._state[StateKey.POSITION], 128) self.assertEqual(self.state_callback_value[StateKey.POSITION], 128) self.assertEqual(self.publish_callback_value[0], self.particle_a_eng_telem) self.assertEqual(self.publish_callback_value[1], self.particle_b_eng_telem) self.assertEqual(self.publish_callback_value[2], self.particle_c_eng_telem) self.assertEqual(self.publish_callback_value[3], self.particle_d_eng_telem) self.assertEqual(self.file_ingested, True)
def test_bad_flags_telemetered(self): """ test that we don't parse any records when the flags are not what we expect """ with self.assertRaises(SampleException): self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_BAD_FLAGS) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback)
def test_bad_flags_telemetered(self): """ test that we don't parse any records when the flags are not what we expect """ file_path = os.path.join(RESOURCE_PATH, 'bad_flags.dat') with self.assertRaises(SampleException): with open(file_path, 'rb') as stream_handle: parser = WfpEngStcImodemParser(self._telem_config, None, stream_handle, lambda state, ingested: None, lambda data: None)
def test_mid_state_start_telemetered(self): """ Test starting the parser in a state in the middle of processing """ new_state = {StateKey.POSITION: 76} self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), new_state, self.stream_handle, self.state_callback, self.pub_callback) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng_telem, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng_telem, True)
def test_simple_telemetered(self): """ Read test data and pull out data particles one at a time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'simple.dat') with open(file_path, 'rb') as stream_handle: parser = WfpEngStcImodemParser(self._telem_config, None, stream_handle, lambda state, ingested: None, lambda data: None) result = parser.get_records(5) self.assert_particles(result, 'simple_telem.yml', RESOURCE_PATH)
def test_bad_data_telemetered(self): """ Ensure that missing data causes us to miss records TODO: This test should be improved if we come up with a more accurate regex for the data sample """ file_path = os.path.join(RESOURCE_PATH, 'bad_data.dat') with open(file_path, 'rb') as stream_handle: parser = WfpEngStcImodemParser(self._telem_config, None, stream_handle, lambda state, ingested: None, lambda data: None) result = parser.get_records(5) self.assert_particles(result, 'bad_data_telem.yml', RESOURCE_PATH)
def test_long_stream_telemetered(self): """ Test a long stream of data """ file_path = os.path.join(RESOURCE_PATH, 'E0000039.DAT') with open(file_path, 'rb') as stream_handle: parser = WfpEngStcImodemParser(self._telem_config, None, stream_handle, lambda state, ingested: None, lambda data: None) # start with the start time record result = parser.get_records(100) self.assert_particles(result, 'E0000039_telem.yml', RESOURCE_PATH)
def test_bad_data_telemetered(self): """ Ensure that missing data causes us to miss records TODO: This test should be improved if we come up with a more accurate regex for the data sample """ self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_BAD_ENG) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(1) self.assert_result(result, 24, self.particle_a_start_time_telem, False) # next get engineering records result = self.parser.get_records(4) if len(result) == 4: self.fail("We got 4 records, the bad data should only make 3")
def test_after_header_recovered(self): """ Test starting the parser in a state in the middle of processing """ new_state = {StateKey.POSITION: 24} self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_RECOVERED), new_state, self.stream_handle, self.state_callback, self.pub_callback) # get engineering records result = self.parser.get_records(1) self.assert_result(result, 50, self.particle_a_eng_recov, False) result = self.parser.get_records(1) self.assert_result(result, 76, self.particle_b_eng_recov, False) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng_recov, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng_recov, True)
def test_bug_3241(self): """ This test was created to validate fixes to bug #3241 It validates the parser can parse a file recovered from Global platforms """ file_path = os.path.join(RESOURCE_PATH, 'E0000000.DAT') with open(file_path, 'rb') as stream_handle: parser = WfpEngStcImodemParser(self._recov_config, None, stream_handle, lambda state, ingested: None, lambda data: None) result = parser.get_records(100) # make sure we get 100 particles back self.assertEquals(len(result), 100) # make sure there are no errors self.assertEquals(len(self.exception_callback_value), 0)
def process(self): log = get_logger() with open(self._sourceFilePath, "rb") as file_handle: def exception_callback(exception): log.debug("Exception: %s", exception) self._particleDataHdlrObj.setParticleDataCaptureFailure() parser = WfpEngStcImodemParser(self._parser_config, None, file_handle, lambda state, ingested: None, lambda data: None, exception_callback) driver = DataSetDriver(parser, self._particleDataHdlrObj) driver.processFileStream() return self._particleDataHdlrObj
def test_set_state_telemetered(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ new_state = {StateKey.POSITION: 76} self.stream_handle = StringIO( WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT) self.parser = WfpEngStcImodemParser( self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED), self.start_state, self.stream_handle, self.state_callback, self.pub_callback) # start with the start time record result = self.parser.get_records(1) self.assert_result(result, 24, self.particle_a_start_time_telem, False) # set the new state, the essentially skips engineering a and b self.parser.set_state(new_state) result = self.parser.get_records(1) self.assert_result(result, 102, self.particle_c_eng_telem, False) result = self.parser.get_records(1) self.assert_result(result, 128, self.particle_d_eng_telem, True)
def _build_parser(self, parser_state, infile, data_key=None): """ Build and return the parser """ config = self._parser_config.get(data_key) # # If the key is WFP_ENG_STC_IMODEM, build the WFP parser. # if data_key == DataTypeKey.WFP_ENG_STC_IMODEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'status_data_particle_class': WfpEngStcImodemStatusRecoveredDataParticle, 'start_data_particle_class': WfpEngStcImodemStartRecoveredDataParticle, 'engineering_data_particle_class': WfpEngStcImodemEngineeringRecoveredDataParticle } }) parser = WfpEngStcImodemParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) # # If the key is WFP_ENG_WFP_SIO_MULE, build the WFP SIO Mule parser. # elif data_key == DataTypeKey.WFP_ENG_WFP_SIO_MULE: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng_wfp_sio_mule', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'start_time_data_particle_class': WfpEngWfpSioMuleParserDataStartTimeParticle, 'status_data_particle_class': WfpEngWfpSioMuleParserDataStatusParticle, 'engineering_data_particle_class': WfpEngWfpSioMuleParserDataEngineeringParticle } }) parser = WfpEngWfpSioMuleParser( config, parser_state, infile, lambda state: self._save_parser_state(state, data_key), self._data_callback, self._sample_exception_callback) # # If the key is one that we're not expecting, don't build any parser. # else: raise ConfigurationException( "Invalid data_key supplied to build parser") return parser