def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
self.stream_handle = StringIO(
Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = Parad_k_stc_imodemParser(self.config, self.start_state,
self.stream_handle,
self.state_callback,
self.pub_callback)
# start with the start time record
result = self.parser.get_records(4)
self.assertEqual(result, [
self.particle_a_eng, self.particle_b_eng, self.particle_c_eng,
self.particle_d_eng
])
self.assertEqual(self.parser._state[StateKey.POSITION], 128)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 128)
self.assertEqual(self.publish_callback_value[0], self.particle_a_eng)
self.assertEqual(self.publish_callback_value[1], self.particle_b_eng)
self.assertEqual(self.publish_callback_value[2], self.particle_c_eng)
self.assertEqual(self.publish_callback_value[3], self.particle_d_eng)
self.assertEqual(self.file_ingested, True)
def parse(unused, source_file_path, particle_data_handler):
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle'
}
def state_callback(state, ingested):
pass
def pub_callback(data):
log.trace("Found data: %s", data)
def exception_callback(exception):
particle_data_handler.setParticleDataCaptureFailure()
stream_handle = open(source_file_path, 'rb')
try:
parser = Parad_k_stc_imodemParser(config, None, stream_handle,
state_callback, pub_callback,
exception_callback)
driver = DataSetDriver(parser, particle_data_handler)
driver.processFileStream()
finally:
stream_handle.close()
return particle_data_handler
def test_simple(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
self.stream_handle = StringIO(
Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT
) #turn into a data stream to look like file ingestion
self.parser = Parad_k_stc_imodemParser(
self.config, self.start_state, self.stream_handle,
self.state_callback,
self.pub_callback) # last one is the link to the data source
# next get engineering records
result = self.parser.get_records(1)
self.assert_result(result, 50, self.particle_a_eng, False)
result = self.parser.get_records(1)
self.assert_result(result, 76, self.particle_b_eng, False)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng, True)
# no data left, dont move the position
result = self.parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.parser._state[StateKey.POSITION], 128)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 128)
self.assert_(isinstance(self.publish_callback_value, list))
self.assertEqual(self.publish_callback_value[0], self.particle_d_eng)
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj):
config.add_configuration(
os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml'))
log = get_logger()
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'Parad_k_stc_imodemDataParticle'
}
def exception_callback(exception):
log.debug("ERROR: %r", exception)
particleDataHdlrObj.setParticleDataCaptureFailure()
with open(sourceFilePath, 'rb') as stream_handle:
parser = Parad_k_stc_imodemParser(parser_config, None, stream_handle,
lambda state, ingested: None,
lambda data: None,
exception_callback)
driver = DataSetDriver(parser, particleDataHdlrObj)
driver.processFileStream()
return particleDataHdlrObj
def test_bad_flags(self):
"""
test that we don't parse any records when the flags are not what we expect
"""
with self.assertRaises(SampleException):
self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_BAD_FLAGS)
self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle,
self.state_callback, self.pub_callback)
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
new_state = {StateKey.POSITION:76}
self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = Parad_k_stc_imodemParser(self.config, new_state, self.stream_handle,
self.state_callback, self.pub_callback)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng, True)
def test_bad_data(self):
"""
Ensure that missing data causes us to miss records
TODO: This test should be improved if we come up with a more accurate regex for the data sample
"""
self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_BAD_ENG)
self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle,
self.state_callback, self.pub_callback)
# next get engineering records
result = self.parser.get_records(4)
if len(result) == 4:
self.fail("We got 4 records, the bad data should only make 3")
def test_long_stream(self):
"""
Test a long stream of data
"""
self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA)
self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle,
self.state_callback, self.pub_callback)
result = self.parser.get_records(32)
self.assertEqual(result[0], self.particle_a_eng)
self.assertEqual(result[-1], self.particle_last_eng)
self.assertEqual(self.parser._state[StateKey.POSITION], 856)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 856)
self.assertEqual(self.publish_callback_value[-1], self.particle_last_eng)
def _build_parser(self, parser_state, infile, data_key=None):
"""
Build and return the specified parser as indicated by the data_key.
"""
#
# If the key is PARAD_K_STC, build the telemetered parser.
#
if data_key == DataTypeKey.PARAD_K_STC:
config = self._parser_config[data_key]
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'Parad_k_stc_imodemDataParticle'
})
log.debug("My Config: %s", config)
parser = Parad_k_stc_imodemParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
#
# If the key is PARAD_K_STC_RECOVERED, build the recovered parser.
#
elif data_key == DataTypeKey.PARAD_K_STC_RECOVERED:
config = self._parser_config[data_key]
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'Parad_k_stc_imodemRecoveredDataParticle'
})
log.debug("My Config: %s", config)
parser = Parad_k_stc_imodemRecoveredParser(
config, parser_state, infile,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException('Parser configuration incorrect %s',
data_key)
return parser
def test_set_state(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
new_state = {StateKey.POSITION:76}
self.stream_handle = StringIO(Parad_k_stc_imodemParserUnitTestCase.TEST_DATA_SHORT)
self.parser = Parad_k_stc_imodemParser(self.config, self.start_state, self.stream_handle,
self.state_callback, self.pub_callback)
# set the new state, the essentially skips engineering a and b
self.parser.set_state(new_state)
result = self.parser.get_records(1)
self.assert_result(result, 102, self.particle_c_eng, False)
result = self.parser.get_records(1)
self.assert_result(result, 128, self.particle_d_eng, True)
def parse(unused, source_file_path, particle_data_handler):
parser_config = {
DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS: 'Parad_k_stc_imodemDataParticle'
}
def exception_callback(exception):
log.debug("ERROR: %r", exception)
particle_data_handler.setParticleDataCaptureFailure()
with open(source_file_path, 'rb') as stream_handle:
parser = Parad_k_stc_imodemParser(parser_config,
None,
stream_handle,
lambda state, ingested: None,
lambda data: None,
exception_callback)
driver = DataSetDriver(parser, particle_data_handler)
driver.processFileStream()
return particle_data_handler
def parse(basePythonCodePath, sourceFilePath, particleDataHdlrObj):
from mi.logging import config
config.add_configuration(
os.path.join(basePythonCodePath, 'res', 'config', 'mi-logging.yml'))
from mi.core.log import get_logger
log = get_logger()
from mi.dataset.dataset_driver import DataSetDriver, ParticleDataHandler
from mi.dataset.parser.parad_k_stc_imodem import \
Parad_k_stc_imodemParser
from mi.dataset.dataset_parser import DataSetDriverConfigKeys
config = {
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.parad_k_stc_imodem',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'Parad_k_stc_imodemDataParticle'
}
try:
if particleDataHdlrObj is not None:
pass
except NameError:
particleDataHdlrObj = ParticleDataHandler()
try:
if sourceFilePath is not None:
pass
except NameError:
try:
sourceFilePath = sys.argv[1]
except IndexError:
print "Need a source file path"
sys.exit(1)
def state_callback(state, ingested):
pass
def pub_callback(data):
log.trace("Found data: %s", data)
def exception_callback(exception):
particleDataHdlrObj.setParticleDataCaptureFailure()
stream_handle = open(sourceFilePath, 'rb')
try:
parser = Parad_k_stc_imodemParser(config, None, stream_handle,
state_callback, pub_callback,
exception_callback)
driver = DataSetDriver(parser, particleDataHdlrObj)
driver.processFileStream()
finally:
stream_handle.close()
stream_handle = open(sourceFilePath, 'rb')
return particleDataHdlrObj
