def test_long_stream(self):
"""
Test a long (normal length file)
"""
self.stream_handle = open(
os.path.join(RESOURCE_PATH, '20140120_140004_extradata.mopak.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140120_140004.mopak.log',
self.state_callback, self.pub_callback,
self.except_callback)
result = self.parser.get_records(11964)
self.assertEqual(result[0], self.particle_a_accel)
self.assertEqual(result[-1], self.particle_last_accel)
self.assertEqual(self.parser._state[StateKey.POSITION], 514452)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 514452)
self.assertEqual(self.parser._state[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.state_callback_value[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.parser._state[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.state_callback_value[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.publish_callback_value[-1],
self.particle_last_accel)
self.assertEqual(self.exception_callback_value, None)
def test_long_stream_rate(self):
"""
Test a long (normal length file) with accel and rate particles
"""
self.stream_handle = open(
os.path.join(RESOURCE_PATH, '20140313_191853.3dmgx3.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140313_191853.3dmgx3.log',
self.state_callback, self.pub_callback,
self.except_callback)
result = self.parser.get_records(148)
self.assertEqual(result[0], self.particle_a11_accel)
self.assertEqual(result[-1], self.particle_last_rate)
self.assertEqual(self.parser._state[StateKey.POSITION], 5560)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 5560)
self.assertEqual(self.parser._state[StateKey.TIMER_START],
self._rate_long_timer_start)
self.assertEqual(self.state_callback_value[StateKey.TIMER_START],
self._rate_long_timer_start)
self.assertEqual(self.parser._state[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.state_callback_value[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.publish_callback_value[-1],
self.particle_last_rate)
self.assertEqual(self.exception_callback_value, None)
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
self.stream_handle = open(
os.path.join(RESOURCE_PATH, 'first.mopak.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140120_140004.mopak.log',
self.state_callback, self.pub_callback,
self.except_callback)
# next get accel records
result = self.parser.get_records(5)
self.assertEqual(result, [
self.particle_a_accel, self.particle_b_accel,
self.particle_c_accel, self.particle_d_accel, self.particle_e_accel
])
self.assertEqual(self.parser._state[StateKey.POSITION], 215)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 215)
self.assertEqual(self.parser._state[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.state_callback_value[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.parser._state[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.state_callback_value[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.publish_callback_value[0], self.particle_a_accel)
self.assertEqual(self.publish_callback_value[1], self.particle_b_accel)
self.assertEqual(self.publish_callback_value[2], self.particle_c_accel)
self.assertEqual(self.publish_callback_value[3], self.particle_d_accel)
self.assertEqual(self.publish_callback_value[4], self.particle_e_accel)
self.assertEqual(self.file_ingested_value, True)
self.assertEqual(self.exception_callback_value, None)
def test_non_data_exception(self):
"""
Test that we get a sample exception from non data being found in the file
"""
self.stream_handle = open(
os.path.join(RESOURCE_PATH, 'noise.mopak.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140120_140004.mopak.log',
self.state_callback, self.pub_callback,
self.except_callback)
# next get accel records
result = self.parser.get_records(5)
self.assertEqual(result, [
self.particle_a_accel, self.particle_b_accel,
self.particle_c_accel, self.particle_d_accel, self.particle_e_accel
])
self.assertEqual(self.parser._state[StateKey.POSITION], 218)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 218)
self.assertEqual(self.parser._state[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.state_callback_value[StateKey.TIMER_START],
self._timer_start)
self.assertEqual(self.parser._state[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.state_callback_value[StateKey.TIMER_ROLLOVER], 0)
self.assertEqual(self.publish_callback_value[0], self.particle_a_accel)
self.assertEqual(self.publish_callback_value[1], self.particle_b_accel)
self.assertEqual(self.publish_callback_value[2], self.particle_c_accel)
self.assertEqual(self.publish_callback_value[3], self.particle_d_accel)
self.assertEqual(self.publish_callback_value[4], self.particle_e_accel)
self.assertEqual(self.file_ingested_value, True)
self.assert_(isinstance(self.exception_callback_value,
SampleException))
def test_simple(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH, 'first.mopak.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140120_140004.mopak.log',
self.exception_callback)
# next get acceleration records
result = parser.get_records(1)
self.assertEqual(result[0], self.particle_a_accel)
self.assertEqual(self.exception_callback_value, [])
result = parser.get_records(1)
self.assertEqual(result[0], self.particle_b_accel)
self.assertEqual(self.exception_callback_value, [])
result = parser.get_records(1)
self.assertEqual(result[0], self.particle_c_accel)
self.assertEqual(self.exception_callback_value, [])
result = parser.get_records(1)
self.assertEqual(result[0], self.particle_d_accel)
self.assertEqual(self.exception_callback_value, [])
result = parser.get_records(1)
self.assertEqual(result[0], self.particle_e_accel)
self.assertEqual(self.exception_callback_value, [])
# no data left, don't move the position
result = parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.exception_callback_value, [])
def test_mid_state_start(self):
"""
Test starting the parser in a state in the middle of processing
"""
new_state = {
StateKey.POSITION: 86,
StateKey.TIMER_ROLLOVER: 0,
StateKey.TIMER_START: self._timer_start
}
self.stream_handle = open(
os.path.join(RESOURCE_PATH, 'first.mopak.log'))
self.parser = MopakODclParser(self.config, new_state,
self.stream_handle,
'20140120_140004.mopak.log',
self.state_callback, self.pub_callback,
self.except_callback)
result = self.parser.get_records(1)
self.assert_result(result, 129, self.particle_c_accel, False,
self._timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 172, self.particle_d_accel, False,
self._timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 215, self.particle_e_accel, True,
self._timer_start)
self.assertEqual(self.exception_callback_value, None)
def test_set_state_rate(self):
"""
Test changing to a new state after initializing the parser and
reading data, as if new data has been found and the state has
changed
"""
new_state = {
StateKey.POSITION: 117,
StateKey.TIMER_ROLLOVER: 0,
StateKey.TIMER_START: self._first_rate_timer_start
}
self.stream_handle = open(
os.path.join(RESOURCE_PATH, 'first_rate.mopak.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140313_191853.3dmgx3.log',
self.state_callback, self.pub_callback,
self.except_callback)
result = self.parser.get_records(1)
self.assert_result(result, 43, self.particle_a1_accel, False,
self._first_rate_timer_start)
# set the new state, the essentially skips b accel, a rate
self.parser.set_state(new_state)
result = self.parser.get_records(1)
self.assert_result(result, 148, self.particle_b_rate, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 179, self.particle_c_rate, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 210, self.particle_d_rate, True,
self._first_rate_timer_start)
self.assertEqual(self.exception_callback_value, None)
def test_long_stream_yml(self):
"""
Verify an entire file against a yaml result file.
"""
with open(os.path.join(RESOURCE_PATH, '20140120_140004.mopak.log'), 'rb') as stream_handle:
parser = MopakODclParser(self.config, stream_handle,
'20140120_140004.mopak.log',
self.exception_callback)
particles = parser.get_records(10)
self.assert_particles(particles, 'first_mopak_recov.result.yml', RESOURCE_PATH)
self.assertEqual(self.exception_callback_value, [])
def test_bad_config(self):
"""
This tests that the parser raises a Configuration Exception if the
required configuration items are not present
"""
stream_handle = open(os.path.join(RESOURCE_PATH, 'first.mopak.log'))
config = {}
with self.assertRaises(ConfigurationException):
MopakODclParser(config, stream_handle, '20140120_140004.mopak.log',
self.exception_callback)
def test_long_stream_rate(self):
"""
Test a long (normal length file) with accel and rate particles
"""
stream_handle = open(
os.path.join(RESOURCE_PATH, '20140313_191853.3dmgx3.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140313_191853.3dmgx3.log',
self.exception_callback)
result = parser.get_records(148)
self.assertEqual(result[0], self.particle_a11_accel)
self.assertEqual(result[-1], self.particle_last_rate)
self.assertEqual(self.exception_callback_value, [])
def test_long_stream(self):
"""
Test a long (normal length file)
"""
stream_handle = open(
os.path.join(RESOURCE_PATH, '20140120_140004_extradata.mopak.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140120_140004.mopak.log',
self.exception_callback)
result = parser.get_records(11964)
self.assertEqual(result[0], self.particle_a_accel)
self.assertEqual(result[-1], self.particle_last_accel)
self.assertEqual(self.exception_callback_value, [])
def test_get_many(self):
"""
Read test data and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
stream_handle = open(os.path.join(RESOURCE_PATH, 'first.mopak.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140120_140004.mopak.log',
self.exception_callback)
# next get accel records
result = parser.get_records(5)
self.assertEqual(result, [
self.particle_a_accel, self.particle_b_accel,
self.particle_c_accel, self.particle_d_accel, self.particle_e_accel
])
self.assertEqual(self.exception_callback_value, [])
def process(self):
log = get_logger()
def exception_callback(exception):
log.debug("ERROR: %r", exception)
self._particleDataHdlrObj.setParticleDataCaptureFailure()
pathList = (self._sourceFilePath.split('/'))
filename = pathList[len(pathList) - 1]
with open(self._sourceFilePath, 'rb') as stream_handle:
parser = MopakODclParser(self._parser_config, stream_handle,
filename, exception_callback)
driver = DataSetDriver(parser, self._particleDataHdlrObj)
driver.processFileStream()
return self._particleDataHdlrObj
def test_non_data_exception(self):
"""
Test that we get a sample exception from non data being found in the file
"""
stream_handle = open(os.path.join(RESOURCE_PATH, 'noise.mopak.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140120_140004.mopak.log',
self.exception_callback)
# next get accel records
result = parser.get_records(5)
self.assertEqual(result, [
self.particle_a_accel, self.particle_b_accel,
self.particle_c_accel, self.particle_d_accel, self.particle_e_accel
])
self.assertEqual(len(self.exception_callback_value), 1)
self.assert_(
isinstance(self.exception_callback_value[0], SampleException))
def test_bad_checksum(self):
"""
This tests that the parser raises a Configuration Exception if the
required configuration items are not present
"""
stream_handle = open(os.path.join(RESOURCE_PATH,
'20140313_191853_bad_chksum.3dmgx3.log'))
parser = MopakODclParser(self.config, stream_handle,
'20140313_191853.3dmgx3.log',
self.exception_callback)
result = parser.get_records(10)
self.assertEqual(len(result), 5)
self.assertEqual(len(self.exception_callback_value), 2)
self.assert_(isinstance(self.exception_callback_value[0], SampleException))
self.assert_(isinstance(self.exception_callback_value[1], SampleException))
def test_simple_rate(self):
"""
Read test data and pull out data particles one at a time.
Assert that the results are those we expected.
"""
self.stream_handle = open(
os.path.join(RESOURCE_PATH, 'first_rate.mopak.log'))
self.parser = MopakODclParser(self.config, self.start_state,
self.stream_handle,
'20140313_191853.mopak.log',
self.state_callback, self.pub_callback,
self.except_callback)
# next get accel and rate records
result = self.parser.get_records(1)
self.assert_result(result, 43, self.particle_a1_accel, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 86, self.particle_b1_accel, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 117, self.particle_a_rate, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 148, self.particle_b_rate, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 179, self.particle_c_rate, False,
self._first_rate_timer_start)
result = self.parser.get_records(1)
self.assert_result(result, 210, self.particle_d_rate, True,
self._first_rate_timer_start)
# no data left, dont move the position
result = self.parser.get_records(1)
self.assertEqual(result, [])
self.assertEqual(self.parser._state[StateKey.POSITION], 210)
self.assertEqual(self.state_callback_value[StateKey.POSITION], 210)
self.assert_(isinstance(self.publish_callback_value, list))
self.assertEqual(self.publish_callback_value[0], self.particle_d_rate)
self.assertEqual(self.exception_callback_value, None)
def _build_parser(self, parser_state, stream_in, data_key, file_in):
"""
Build the parser based on which data_key is input. The file name is only
needed for mopak, and it just not passed in to the other parser builders
@param parser_state previous parser state to initialize parser with
@param stream_in handle of the opened file to parse
@param data_key harvester / parser key
@param file_in file name
"""
# get the config for the correct parser instance
config = self._parser_config.get(data_key)
if config is None:
log.warn(
'Parser config does not exist for key = %s. Not building parser',
data_key)
raise ConfigurationException
if data_key == DataTypeKey.CG_STC_ENG_TELEM:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.cg_stc_eng_stc',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'CgStcEngStcParserDataParticle'
})
parser = CgStcEngStcParser(
config, parser_state, stream_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
elif data_key == DataTypeKey.CG_STC_ENG_RECOV:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.cg_stc_eng_stc',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'CgStcEngStcParserRecoveredDataParticle'
})
parser = CgStcEngStcParser(
config, parser_state, stream_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
elif data_key == DataTypeKey.MOPAK_TELEM:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.mopak_o_dcl',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
# particle_class configuration does nothing for multi-particle parsers
# put the class names in specific config parameters so the parser can get them
# use real classes as objects instead of strings to make it easier
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
MopakParticleClassType.ACCEL_PARTCICLE_CLASS:
MopakODclAccelParserDataParticle,
MopakParticleClassType.RATE_PARTICLE_CLASS:
MopakODclRateParserDataParticle
}
})
parser = MopakODclParser(
config, parser_state, stream_in, file_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
elif data_key == DataTypeKey.MOPAK_RECOV:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.mopak_o_dcl',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
# particle_class configuration does nothing for multi-particle parsers
# put the class names in specific config parameters so the parser can get them
# use real classes as objects instead of strings to make it easier
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
MopakParticleClassType.ACCEL_PARTCICLE_CLASS:
MopakODclAccelParserRecoveredDataParticle,
MopakParticleClassType.RATE_PARTICLE_CLASS:
MopakODclRateParserRecoveredDataParticle
}
})
parser = MopakODclParser(
config, parser_state, stream_in, file_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
elif data_key == DataTypeKey.RTE_TELEM:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.rte_o_dcl',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'RteODclParserDataParticle'
})
parser = RteODclParser(
config, parser_state, stream_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
elif data_key == DataTypeKey.RTE_RECOV:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.rte_o_dcl',
DataSetDriverConfigKeys.PARTICLE_CLASS:
'RteODclParserRecoveredDataParticle'
})
parser = RteODclParser(
config, parser_state, stream_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
else:
log.warn('Invalid Data_Key %s. Not building parser', data_key)
raise ConfigurationException
return parser
