def test_mid_state_start_recovered(self): """ test starting a parser with a state in the middle of processing """ new_state = { StateKey.IN_PROCESS_DATA: [], StateKey.UNPROCESSED_DATA: [[117, 6300]], StateKey.METADATA_SENT: True, StateKey.FILE_SIZE: 9400 } stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), new_state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) self.assert_result( result, [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]], [[235, 6300]], self.particle_rb) result = self.parser.get_records(2) self.assertEqual(result[0], self.particle_rc) self.assertEqual(result[1], self.particle_rd) self.assert_state( [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]], [[469, 6300]]) stream_handle.close() self.assertEqual(self.exception_callback_value, None)
def test_get_many_recovered(self): """ Read test data from the file and pull out multiple data particles at one time. Assert that the results are those we expected. """ state = { StateKey.UNPROCESSED_DATA: [[0, 1000]], StateKey.IN_PROCESS_DATA: [], StateKey.METADATA_SENT: False, StateKey.FILE_SIZE: 9400 } stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) result = self.parser.get_records(4) stream_handle.close() self.assertEqual(result, [ self.particle_rmetadata, self.particle_ra, self.particle_rb, self.particle_rc ]) self.assert_state([[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]], [[352, 1000]]) self.assertEqual(self.publish_callback_value[0], self.particle_rmetadata) self.assertEqual(self.publish_callback_value[1], self.particle_ra) self.assertEqual(self.publish_callback_value[2], self.particle_rb) self.assertEqual(self.publish_callback_value[3], self.particle_rc) self.assertEqual(self.exception_callback_value, None)
def _build_parser(self, parser_state, stream_in, data_key=None): """ Build and return the parser """ config = self._parser_config.get(data_key) # # If the key is DOSTA_ABCDJM_SIO_RECOVERED, build the WFP parser. # if data_key == DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dostad', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { METADATA_PARTICLE_CLASS_KEY: DostadParserRecoveredMetadataDataParticle, DATA_PARTICLE_CLASS_KEY: DostadParserRecoveredDataParticle } }) parser = DostadRecoveredParser( config, parser_state, stream_in, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) return parser # # If the key is DOSTA_ABCDJM_SIO_TELEMETERED, build the WFP SIO Mule parser. # elif data_key == DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.dostad', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { METADATA_PARTICLE_CLASS_KEY: DostadParserTelemeteredMetadataDataParticle, DATA_PARTICLE_CLASS_KEY: DostadParserTelemeteredDataParticle } }) parser = DostadParser( config, parser_state, stream_in, lambda state: self._save_parser_state(state, data_key), self._data_callback, self._sample_exception_callback) return parser # # If the key is one that we're not expecting, don't build any parser. # else: raise ConfigurationException( "Invalid data_key supplied to build parser")
def test_simple_recovered(self): stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) # NOTE: using the unprocessed data state of 0,6300 limits the file to reading # just 6300 bytes, so even though the file is longer it only reads the first # 6300 state = { StateKey.UNPROCESSED_DATA: [[0, 6300]], StateKey.IN_PROCESS_DATA: [], StateKey.METADATA_SENT: False, StateKey.FILE_SIZE: 9400 } self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) in_process = [[0, 117, 2, 1], [117, 235, 1, 0], [235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[0, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rmetadata) self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True) # then get the first dosta data particle, this clears out the block from 197-314 result = self.parser.get_records(1) in_process = [[117, 235, 1, 0], [235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[117, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_ra) self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True) result = self.parser.get_records(1) in_process = [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[235, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rb) result = self.parser.get_records(1) in_process = [[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[352, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rc) result = self.parser.get_records(1) in_process = [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[469, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rd) stream_handle.close() self.assertEqual(self.exception_callback_value, None)
def test_long_stream_recovered(self): stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), None, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) result = self.parser.get_records(7) stream_handle.close() self.assertEqual(result[0], self.particle_rmetadata) self.assertEqual(result[1], self.particle_ra) self.assertEqual(result[2], self.particle_rb) self.assertEqual(result[3], self.particle_rc) self.assertEqual(result[4], self.particle_rd) self.assertEqual(result[5], self.particle_re) self.assertEqual(result[6], self.particle_rf) self.assert_state([[703, 820, 1, 0]], [[703, 822]]) self.assertEqual(self.publish_callback_value[5], self.particle_re) self.assertEqual(self.publish_callback_value[6], self.particle_rf) self.assertEqual(self.exception_callback_value, None)
def test_set_state_recovered(self): """ test changing the state after initializing """ state = { StateKey.UNPROCESSED_DATA: [[0, 352]], StateKey.IN_PROCESS_DATA: [], StateKey.METADATA_SENT: False, StateKey.FILE_SIZE: 9400 } new_state = { StateKey.UNPROCESSED_DATA: [[352, 6300]], StateKey.IN_PROCESS_DATA: [], StateKey.METADATA_SENT: True, StateKey.FILE_SIZE: 9400 } stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) # there should only be 4 records, make sure we stop there result = self.parser.get_records(4) self.assert_state([], []) result = self.parser.get_records(1) self.assertEqual(result, []) self.parser.set_state(new_state) result = self.parser.get_records(1) stream_handle.close() self.assert_result( result, [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]], [[469, 6300]], self.particle_rd) self.assertEqual(self.exception_callback_value, None)
def test_update_recovered(self): """ Test a file which has had a section of data replaced by 0s, as if a block of data has not been received yet, then using the returned state make a new parser with the test data that has the 0s filled in """ state = { StateKey.UNPROCESSED_DATA: [[0, 6300]], StateKey.IN_PROCESS_DATA: [], StateKey.METADATA_SENT: False, StateKey.FILE_SIZE: 9400 } # this file has a block of DO data replaced by 0s stream_handle = open( os.path.join(RESOURCE_PATH, 'DOS15908_1st7_replaced.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) result = self.parser.get_records(1) in_process = [[0, 117, 2, 1], [235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[0, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rmetadata) self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True) # then get the first dosta data particle, this clears out the block from 197-314 result = self.parser.get_records(1) in_process = [[235, 352, 1, 0], [352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[117, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_ra) self.assertEqual(self.parser._state[StateKey.METADATA_SENT], True) result = self.parser.get_records(1) in_process = [[352, 469, 1, 0], [469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[117, 235], [352, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rc) stream_handle.close() next_state = self.parser._state # this file has the block of data that was missing in the previous file stream_handle = open(os.path.join(RESOURCE_PATH, 'DOS15908_1st7.DAT')) self.parser = DostadRecoveredParser( self.config.get(DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), next_state, stream_handle, self.state_callback_recovered, self.pub_callback, self.exception_callback) # first get the old 'in process' records from 6131-6248 # Once those are done, the un processed data will be checked result = self.parser.get_records(1) in_process = [[469, 586, 1, 0], [586, 703, 1, 0], [703, 820, 1, 0]] unprocessed = [[117, 235], [469, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rd) # 3 records remain, then the subsequent read should find the previously missing record result = self.parser.get_records(3) result = self.parser.get_records(1) in_process = [] unprocessed = [[820, 6300]] self.assert_result(result, in_process, unprocessed, self.particle_rb) stream_handle.close() self.assertEqual(self.exception_callback_value, None)