def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
# Default the parser to None
parser = None
config = self._parser_config.get(data_key)
#
# If the key is WFP_ENG_STC_IMODEM_RECOVERED, build the Wfp_eng__stc_imodemParser parser and
# provide a config that includes the specific recovered particle types.
#
if data_key == DataTypeKey.WFP_ENG_STC_IMODEM_RECOVERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'status_data_particle_class': WfpEngStcImodemStatusRecoveredDataParticle,
'start_data_particle_class': WfpEngStcImodemStartRecoveredDataParticle,
'engineering_data_particle_class': WfpEngStcImodemEngineeringRecoveredDataParticle
}
})
log.debug("My Config: %s", config)
parser = WfpEngStcImodemParser(
config,
parser_state,
infile,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback)
#
# If the key is WFP_ENG_STC_IMODEM_TELEMETERED, build the Wfp_eng__stc_imodemParser parser and
# provide a config that includes the specific telemetered particle types.
#
elif data_key == DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'status_data_particle_class': WfpEngStcImodemStatusTelemeteredDataParticle,
'start_data_particle_class': WfpEngStcImodemStartTelemeteredDataParticle,
'engineering_data_particle_class': WfpEngStcImodemEngineeringTelemeteredDataParticle
}
})
log.debug("My Config: %s", config)
parser = WfpEngStcImodemParser(
config,
parser_state,
infile,
lambda state, ingested: self._save_parser_state(state, data_key, ingested),
self._data_callback,
self._sample_exception_callback)
else:
raise ConfigurationException
return parser
def test_long_stream_telemetered(self):
"""
Test a long stream of data
"""
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(1)
self.assert_result(result, 24, self.particle_a_start_time_telem, False)
result = self.parser.get_records(32)
self.assertEqual(result[0], self.particle_a_eng_telem)
self.assertEqual(result[-1], self.particle_last_eng_telem)
self.assertEqual(self.parser._state[StateKey.POSITION], 856)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 856)
self.assertEqual(self.publish_callback_value[-1],
self.particle_last_eng_telem)
result = self.parser.get_records(1)
self.assert_result(result, 872, self.particle_a_status_telem, True)
def test_simple_telemetered(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(1)
self.assert_result(result, 24, self.particle_a_start_time_telem, False)
# next get engineering records
result = self.parser.get_records(1)
self.assert_result(result, 50, self.particle_a_eng_telem, False)
result = self.parser.get_records(1)
self.assert_result(result, 76, self.particle_b_eng_telem, False)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng_telem, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng_telem, True)
# no data left, dont move the position
result = self.parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.parser._state[StateKey.POSITION], 128)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 128)
self.assert_(isinstance(self.publish_callback_value, list))
self.assertEqual(self.publish_callback_value[0],
self.particle_d_eng_telem)
def test_get_many_telemetered(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(1)
self.assert_result(result, 24, self.particle_a_start_time_telem, False)
result = self.parser.get_records(4)
self.assertEqual(result, [
self.particle_a_eng_telem, self.particle_b_eng_telem,
self.particle_c_eng_telem, self.particle_d_eng_telem
])
self.assertEqual(self.parser._state[StateKey.POSITION], 128)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 128)
self.assertEqual(self.publish_callback_value[0],
self.particle_a_eng_telem)
self.assertEqual(self.publish_callback_value[1],
self.particle_b_eng_telem)
self.assertEqual(self.publish_callback_value[2],
self.particle_c_eng_telem)
self.assertEqual(self.publish_callback_value[3],
self.particle_d_eng_telem)
self.assertEqual(self.file_ingested, True)
def test_bad_flags_telemetered(self):
"""
test that we don't parse any records when the flags are not what we expect
"""
with self.assertRaises(SampleException):
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_BAD_FLAGS)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
def test_bad_flags_telemetered(self):
"""
test that we don't parse any records when the flags are not what we expect
"""
file_path = os.path.join(RESOURCE_PATH, 'bad_flags.dat')
with self.assertRaises(SampleException):
with open(file_path, 'rb') as stream_handle:
parser = WfpEngStcImodemParser(self._telem_config, None,
stream_handle,
lambda state, ingested: None,
lambda data: None)
def test_mid_state_start_telemetered(self):
"""
Test starting the parser in a state in the middle of processing
"""
new_state = {StateKey.POSITION: 76}
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
new_state, self.stream_handle, self.state_callback,
self.pub_callback)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng_telem, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng_telem, True)
def test_simple_telemetered(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
file_path = os.path.join(RESOURCE_PATH, 'simple.dat')
with open(file_path, 'rb') as stream_handle:
parser = WfpEngStcImodemParser(self._telem_config, None,
stream_handle,
lambda state, ingested: None,
lambda data: None)
result = parser.get_records(5)
self.assert_particles(result, 'simple_telem.yml', RESOURCE_PATH)
def test_bad_data_telemetered(self):
"""
Ensure that missing data causes us to miss records
TODO: This test should be improved if we come up with a more accurate regex for the data sample
"""
file_path = os.path.join(RESOURCE_PATH, 'bad_data.dat')
with open(file_path, 'rb') as stream_handle:
parser = WfpEngStcImodemParser(self._telem_config, None,
stream_handle,
lambda state, ingested: None,
lambda data: None)
result = parser.get_records(5)
self.assert_particles(result, 'bad_data_telem.yml', RESOURCE_PATH)
def test_long_stream_telemetered(self):
"""
Test a long stream of data
"""
file_path = os.path.join(RESOURCE_PATH, 'E0000039.DAT')
with open(file_path, 'rb') as stream_handle:
parser = WfpEngStcImodemParser(self._telem_config, None,
stream_handle,
lambda state, ingested: None,
lambda data: None)
# start with the start time record
result = parser.get_records(100)
self.assert_particles(result, 'E0000039_telem.yml', RESOURCE_PATH)
def test_bad_data_telemetered(self):
"""
Ensure that missing data causes us to miss records
TODO: This test should be improved if we come up with a more accurate regex for the data sample
"""
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_BAD_ENG)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(1)
self.assert_result(result, 24, self.particle_a_start_time_telem, False)
# next get engineering records
result = self.parser.get_records(4)
if len(result) == 4:
self.fail("We got 4 records, the bad data should only make 3")
def test_after_header_recovered(self):
"""
Test starting the parser in a state in the middle of processing
"""
new_state = {StateKey.POSITION: 24}
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_RECOVERED),
new_state, self.stream_handle, self.state_callback,
self.pub_callback)
# get engineering records
result = self.parser.get_records(1)
self.assert_result(result, 50, self.particle_a_eng_recov, False)
result = self.parser.get_records(1)
self.assert_result(result, 76, self.particle_b_eng_recov, False)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng_recov, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng_recov, True)
def test_bug_3241(self):
"""
This test was created to validate fixes to bug #3241
It validates the parser can parse a file recovered from
Global platforms
"""
file_path = os.path.join(RESOURCE_PATH, 'E0000000.DAT')
with open(file_path, 'rb') as stream_handle:
parser = WfpEngStcImodemParser(self._recov_config, None,
stream_handle,
lambda state, ingested: None,
lambda data: None)
result = parser.get_records(100)
# make sure we get 100 particles back
self.assertEquals(len(result), 100)
# make sure there are no errors
self.assertEquals(len(self.exception_callback_value), 0)
def process(self):
log = get_logger()
with open(self._sourceFilePath, "rb") as file_handle:
def exception_callback(exception):
log.debug("Exception: %s", exception)
self._particleDataHdlrObj.setParticleDataCaptureFailure()
parser = WfpEngStcImodemParser(self._parser_config, None,
file_handle,
lambda state, ingested: None,
lambda data: None,
exception_callback)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def test_set_state_telemetered(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
new_state = {StateKey.POSITION: 76}
self.stream_handle = StringIO(
WfpEngStcImodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = WfpEngStcImodemParser(
self.config.get(DataTypeKey.WFP_ENG_STC_IMODEM_TELEMETERED),
self.start_state, self.stream_handle, self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(1)
self.assert_result(result, 24, self.particle_a_start_time_telem, False)
# set the new state, the essentially skips engineering a and b
self.parser.set_state(new_state)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng_telem, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng_telem, True)
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the parser
"""
config = self._parser_config.get(data_key)
#
# If the key is WFP_ENG_STC_IMODEM, build the WFP parser.
#
if data_key == DataTypeKey.WFP_ENG_STC_IMODEM:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.wfp_eng__stc_imodem_particles',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'status_data_particle_class':
WfpEngStcImodemStatusRecoveredDataParticle,
'start_data_particle_class':
WfpEngStcImodemStartRecoveredDataParticle,
'engineering_data_particle_class':
WfpEngStcImodemEngineeringRecoveredDataParticle
}
})
parser = WfpEngStcImodemParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
#
# If the key is WFP_ENG_WFP_SIO_MULE, build the WFP SIO Mule parser.
#
elif data_key == DataTypeKey.WFP_ENG_WFP_SIO_MULE:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.wfp_eng_wfp_sio_mule',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
'start_time_data_particle_class':
WfpEngWfpSioMuleParserDataStartTimeParticle,
'status_data_particle_class':
WfpEngWfpSioMuleParserDataStatusParticle,
'engineering_data_particle_class':
WfpEngWfpSioMuleParserDataEngineeringParticle
}
})
parser = WfpEngWfpSioMuleParser(
config, parser_state, infile,
lambda state: self._save_parser_state(state, data_key),
self._data_callback, self._sample_exception_callback)
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException(
"Invalid data_key supplied to build parser")
return parser
