def test_mid_state_start_recovered(self):
"""
test starting a parser with a state in the middle of processing
"""
new_state = {
StateKey.IN_PROCESS_DATA: [],
StateKey.UNPROCESSED_DATA: [[117, 6300]],
StateKey.METADATA_SENT: True,
StateKey.FILE_SIZE: 9400
}
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), new_state,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
result = self.parser.get_records(1)
self.assert_result(
result, [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0],
[586, 703, 1, 0], [703, 820, 1, 0]], [[235, 6300]],
self.particle_rb)
result = self.parser.get_records(2)
self.assertEqual(result[0], self.particle_rc)
self.assertEqual(result[1], self.particle_rd)
self.assert_state(
[[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]],
[[469, 6300]])
stream_handle.close()
self.assertEqual(self.exception_callback_value, None)
def test_get_many_recovered(self):
"""
Read test data from the file and pull out multiple data particles at one time.
Assert that the results are those we expected.
"""
state = {
StateKey.UNPROCESSED_DATA: [[0, 1000]],
StateKey.IN_PROCESS_DATA: [],
StateKey.METADATA_SENT: False,
StateKey.FILE_SIZE: 9400
}
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
result = self.parser.get_records(4)
stream_handle.close()
self.assertEqual(result, [
self.particle_rmetadata, self.particle_ra, self.particle_rb,
self.particle_rc
])
self.assert_state([[352, 469, 1, 0], [469, 586, 1, 0],
[586, 703, 1, 0], [703, 820, 1, 0]], [[352, 1000]])
self.assertEqual(self.publish_callback_value[0],
self.particle_rmetadata)
self.assertEqual(self.publish_callback_value[1], self.particle_ra)
self.assertEqual(self.publish_callback_value[2], self.particle_rb)
self.assertEqual(self.publish_callback_value[3], self.particle_rc)
self.assertEqual(self.exception_callback_value, None)
def _build_parser(self, parser_state, stream_in, data_key=None):
"""
Build and return the parser
"""
config = self._parser_config.get(data_key)
#
# If the key is DOSTA_ABCDJM_SIO_RECOVERED, build the WFP parser.
#
if data_key == DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dostad',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
DostadParserRecoveredMetadataDataParticle,
DATA_PARTICLE_CLASS_KEY: DostadParserRecoveredDataParticle
}
})
parser = DostadRecoveredParser(
config, parser_state, stream_in,
lambda state, ingested: self._save_parser_state(
state, data_key, ingested), self._data_callback,
self._sample_exception_callback)
return parser
#
# If the key is DOSTA_ABCDJM_SIO_TELEMETERED, build the WFP SIO Mule parser.
#
elif data_key == DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED:
config.update({
DataSetDriverConfigKeys.PARTICLE_MODULE:
'mi.dataset.parser.dostad',
DataSetDriverConfigKeys.PARTICLE_CLASS: None,
DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
METADATA_PARTICLE_CLASS_KEY:
DostadParserTelemeteredMetadataDataParticle,
DATA_PARTICLE_CLASS_KEY:
DostadParserTelemeteredDataParticle
}
})
parser = DostadParser(
config, parser_state, stream_in,
lambda state: self._save_parser_state(state, data_key),
self._data_callback, self._sample_exception_callback)
return parser
#
# If the key is one that we're not expecting, don't build any parser.
#
else:
raise ConfigurationException(
"Invalid data_key supplied to build parser")
def test_simple_recovered(self):
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
# NOTE: using the unprocessed data state of 0,6300 limits the file to reading
# just 6300 bytes, so even though the file is longer it only reads the first
# 6300
state = {
StateKey.UNPROCESSED_DATA: [[0, 6300]],
StateKey.IN_PROCESS_DATA: [],
StateKey.METADATA_SENT: False,
StateKey.FILE_SIZE: 9400
}
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
result = self.parser.get_records(1)
in_process = [[0, 117, 2, 1], [117, 235, 1, 0], [235, 352, 1, 0],
[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0],
[703, 820, 1, 0]]
unprocessed = [[0, 6300]]
self.assert_result(result, in_process, unprocessed,
self.particle_rmetadata)
self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
# then get the first dosta data particle, this clears out the block from 197-314
result = self.parser.get_records(1)
in_process = [[117, 235, 1, 0], [235, 352, 1, 0], [352, 469, 1, 0],
[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[117, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_ra)
self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
result = self.parser.get_records(1)
in_process = [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0],
[586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[235, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rb)
result = self.parser.get_records(1)
in_process = [[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0],
[703, 820, 1, 0]]
unprocessed = [[352, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rc)
result = self.parser.get_records(1)
in_process = [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[469, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rd)
stream_handle.close()
self.assertEqual(self.exception_callback_value, None)
def test_long_stream_recovered(self):
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), None,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
result = self.parser.get_records(7)
stream_handle.close()
self.assertEqual(result[0], self.particle_rmetadata)
self.assertEqual(result[1], self.particle_ra)
self.assertEqual(result[2], self.particle_rb)
self.assertEqual(result[3], self.particle_rc)
self.assertEqual(result[4], self.particle_rd)
self.assertEqual(result[5], self.particle_re)
self.assertEqual(result[6], self.particle_rf)
self.assert_state([[703, 820, 1, 0]], [[703, 822]])
self.assertEqual(self.publish_callback_value[5], self.particle_re)
self.assertEqual(self.publish_callback_value[6], self.particle_rf)
self.assertEqual(self.exception_callback_value, None)
def test_set_state_recovered(self):
"""
test changing the state after initializing
"""
state = {
StateKey.UNPROCESSED_DATA: [[0, 352]],
StateKey.IN_PROCESS_DATA: [],
StateKey.METADATA_SENT: False,
StateKey.FILE_SIZE: 9400
}
new_state = {
StateKey.UNPROCESSED_DATA: [[352, 6300]],
StateKey.IN_PROCESS_DATA: [],
StateKey.METADATA_SENT: True,
StateKey.FILE_SIZE: 9400
}
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
# there should only be 4 records, make sure we stop there
result = self.parser.get_records(4)
self.assert_state([], [])
result = self.parser.get_records(1)
self.assertEqual(result, [])
self.parser.set_state(new_state)
result = self.parser.get_records(1)
stream_handle.close()
self.assert_result(
result, [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]],
[[469, 6300]], self.particle_rd)
self.assertEqual(self.exception_callback_value, None)
def test_update_recovered(self):
"""
Test a file which has had a section of data replaced by 0s, as if a block of data has not been received yet,
then using the returned state make a new parser with the test data that has the 0s filled in
"""
state = {
StateKey.UNPROCESSED_DATA: [[0, 6300]],
StateKey.IN_PROCESS_DATA: [],
StateKey.METADATA_SENT: False,
StateKey.FILE_SIZE: 9400
}
# this file has a block of DO data replaced by 0s
stream_handle = open(
os.path.join(RESOURCE_PATH, 'DOS15908_1st7_replaced.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state,
stream_handle, self.state_callback_recovered, self.pub_callback,
self.exception_callback)
result = self.parser.get_records(1)
in_process = [[0, 117, 2, 1], [235, 352, 1, 0], [352, 469, 1, 0],
[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[0, 6300]]
self.assert_result(result, in_process, unprocessed,
self.particle_rmetadata)
self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
# then get the first dosta data particle, this clears out the block from 197-314
result = self.parser.get_records(1)
in_process = [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0],
[586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[117, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_ra)
self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True)
result = self.parser.get_records(1)
in_process = [[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0],
[703, 820, 1, 0]]
unprocessed = [[117, 235], [352, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rc)
stream_handle.close()
next_state = self.parser._state
# this file has the block of data that was missing in the previous file
stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT'))
self.parser = DostadRecoveredParser(
self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED),
next_state, stream_handle, self.state_callback_recovered,
self.pub_callback, self.exception_callback)
# first get the old 'in process' records from 6131-6248
# Once those are done, the un processed data will be checked
result = self.parser.get_records(1)
in_process = [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]]
unprocessed = [[117, 235], [469, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rd)
# 3 records remain, then the subsequent read should find the previously missing record
result = self.parser.get_records(3)
result = self.parser.get_records(1)
in_process = []
unprocessed = [[820, 6300]]
self.assert_result(result, in_process, unprocessed, self.particle_rb)
stream_handle.close()
self.assertEqual(self.exception_callback_value, None)
