def test_long_stream(self): """ Test a long stream """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 200000 particles particles = self.parser.get_records(20000) # There is NOT 20000 in the file. Make sure we obtained 11456 particles. self.assertTrue(len(particles) == 11456) # Close the file stream as we don't need it anymore stream_handle.close()
def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) log.info(len(particles)) # Make sure we obtained 30 particles self.assertTrue(len(particles) == 30) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_eng_particle1, particles[28]) # Attempt to retrieve 50 particles particles = self.parser.get_records(50) # Make sure we obtained 50 particles self.assertTrue(len(particles) == 50) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_start_particle1, particles[48]) stream_handle.close()
def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Init the state to declare the UNPROCESSED_DATA as one WE chunk. Declare the IN_PROCESS_DATA as the one WE # chunk that has 5 samples with 1 of the samples as returned. initial_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 1]] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 1 particle self.assertTrue(len(particles) == 1) # Compare the particle to its expected result self.assert_result(self.test_status_particle1, particles[0]) stream_handle.close()
def test_long_stream(self): """ Test a long stream """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 200000 particles particles = self.parser.get_records(20000) # There is NOT 20000 in the file. Make sure we obtained 11456 particles. self.assertTrue(len(particles) == 11456) # Close the file stream as we don't need it anymore stream_handle.close()
def test_simple(self): """ Read test data and pull out 5 data particles. Then examine each particle against its expected result. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 5 particles particles = self.parser.get_records(5) # Close the file stream as we don't need it anymore stream_handle.close() # Make sure we obtained 5 particles self.assertTrue(len(particles) == 5) # Obtain the expected 5 samples from a yml file test_data = self.get_dict_from_yml('first_five.yml') index = 0 for particle in particles: log.info(particle.generate_dict()) # Make sure each retrieved sample matches its expected result self.assert_result(test_data['data'][index], particles[index]) index += 1
def test_start_stop_set_state_and_resume(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Create a new state to declare the UNPROCESSED_DATA as one chunk. Declare the IN_PROCESS_DATA as one chunk # that has 5 samples with 4 of the samples as returned. new_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 4]], StateKey.FILE_SIZE: stat_info.st_size } # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we only obtained 1 particle since the sample test file only holds one WE chunk with 5 total # samples with 4 already returned. self.assertTrue(len(particles) == 1) # Compare the retrieved particle to its expected result. self.assert_result(self.test_eng_particle2, particles[0]) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 0 particles self.assertTrue(len(particles) == 0) # Close the file stream as we don't need it anymore stream_handle.close()
def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as one WE chunk. Declare the IN_PROCESS_DATA as the one WE # chunk that has 5 samples with 1 of the samples as returned. initial_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 1]], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 1 particle self.assertTrue(len(particles) == 1) # Compare the particle to its expected result self.assert_result(self.test_status_particle1, particles[0]) stream_handle.close()
class WfpEngWfpSioMuleParserUnitTestCase(ParserUnitTestCase): """ wfp_eng_wfp_sio_mule Parser unit test suite """ def state_callback(self, state): """ Call back method to watch what comes in via the position callback """ self.state_callback_value = state def pub_callback(self, pub): """ Call back method to watch what comes in via the publish callback """ self.publish_callback_value = pub def exception_callback(self, exception): """ Call back method to match what comes in via the exception callback """ self.exception_callback_value = exception def setUp(self): ParserUnitTestCase.setUp(self) self.config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng_wfp_sio_mule', DataSetDriverConfigKeys.PARTICLE_CLASS: None } # Define test data particles and their associated timestamps which will be # compared with returned results self.file_ingested_value = None self.state_callback_value = None self.publish_callback_value = None self.test_eng_particle1 = {'internal_timestamp': 3583889968.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_TIMESTAMP: 1374901168, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_CURRENT: 295.3860778808594, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_VOLTAGE: 10.81559944152832, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_PRESSURE: 2335.27001953125} self.test_eng_particle2 = {'internal_timestamp': 3583825326.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_TIMESTAMP: 1374836526, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_CURRENT: 0.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_VOLTAGE: 11.584400177001953, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_PRESSURE: 0.0} self.test_start_particle1 = {'internal_timestamp': 3583954925.0, WfpEngWfpSioMuleParserDataStartTimeParticleKey.CONTROLLER_TIMESTAMP: 1374973536, WfpEngWfpSioMuleParserDataStartTimeParticleKey.WFP_PROFILE_START: 1374966125, WfpEngWfpSioMuleParserDataStartTimeParticleKey.WFP_SENSOR_START: 1374966002} self.test_status_particle1 = {'internal_timestamp': 3583826263.0, WfpEngWfpSioMuleParserDataStatusParticleKey.CONTROLLER_TIMESTAMP: 1374837724, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_INDICATOR: -1, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_RAMP_STATUS: 0, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_PROFILE_STATUS: 14, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_PROFILE_STOP: 1374837463, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_SENSOR_STOP: 1374837591, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_DECIMATION_FACTOR: None} def test_simple(self): """ Read test data and pull out 5 data particles. Then examine each particle against its expected result. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 5 particles particles = self.parser.get_records(5) # Close the file stream as we don't need it anymore stream_handle.close() # Make sure we obtained 5 particles self.assertTrue(len(particles) == 5) # Obtain the expected 5 samples from a yml file test_data = self.get_dict_from_yml('first_five.yml') index = 0 for particle in particles: log.info(particle.generate_dict()) # Make sure each retrieved sample matches its expected result self.assert_result(test_data['data'][index], particles[index]) index += 1 def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) log.info(len(particles)) # Make sure we obtained 30 particles self.assertTrue(len(particles) == 30) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_eng_particle1, particles[28]) # Attempt to retrieve 50 particles particles = self.parser.get_records(50) # Make sure we obtained 50 particles self.assertTrue(len(particles) == 50) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_start_particle1, particles[48]) stream_handle.close() def test_long_stream(self): """ Test a long stream """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 200000 particles particles = self.parser.get_records(20000) # There is NOT 20000 in the file. Make sure we obtained 11456 particles. self.assertTrue(len(particles) == 11456) # Close the file stream as we don't need it anymore stream_handle.close() def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Init the state to declare the UNPROCESSED_DATA as one WE chunk. Declare the IN_PROCESS_DATA as the one WE # chunk that has 5 samples with 1 of the samples as returned. initial_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 1]] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 1 particle self.assertTrue(len(particles) == 1) # Compare the particle to its expected result self.assert_result(self.test_status_particle1, particles[0]) stream_handle.close() def test_start_stop_set_state_and_resume(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Create a new state to declare the UNPROCESSED_DATA as one chunk. Declare the IN_PROCESS_DATA as one chunk # that has 5 samples with 4 of the samples as returned. new_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 4]] } # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we only obtained 1 particle since the sample test file only holds one WE chunk with 5 total # samples with 4 already returned. self.assertTrue(len(particles) == 1) # Compare the retrieved particle to its expected result. self.assert_result(self.test_eng_particle2, particles[0]) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 0 particles self.assertTrue(len(particles) == 0) # Close the file stream as we don't need it anymore stream_handle.close() def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, 'node58p3.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 3 particles particles = self.parser.get_records(3) # Make sure we obtained 3 particles self.assertTrue(len(particles) == 3) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes two chunks with one chunk having 5 samples with 3 returned and the other # chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 4 chunks. self.assert_state([[2818, 2982, 5, 3], [4059, 4673, 20, 0]], [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]]) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) # Make sure we obtained 22 particles self.assertTrue(len(particles) == 22) # Make sure the parser's new state indicates that there 3 remaining chunks in the UNPROCESSED_DATA, none of # which have WE samples. self.assert_state([], [[4058, 4059], [7423, 7424], [7594, 7623]]) # Create a new state to declare the UNPROCESSED_DATA including 4 chunks. Declare the IN_PROCESS_DATA as # the two chunks with WE samples one having 5 samples with 3 returned and the other having 20 samples with # 0 returned. new_state = {StateKey.UNPROCESSED_DATA: [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 3], [4059, 4673, 20, 0]]} # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes one chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 3 chunks within one corresponding to the chunk with WE samples. The # other two chunks contain invalid data. self.assert_state([[4059, 4673, 20, 0]], [[4058, 4673], [7423, 7424], [7594, 7623]]) # Close the file stream as we don't need it anymore stream_handle.close() def assert_result(self, test, particle): """ Suite of tests to run against each returned particle and expected results of the same. The test parameter should be a dictionary that contains the keys to be tested in the particle the 'internal_timestamp' and 'position' keys are treated differently than others but can be verified if supplied """ particle_dict = particle.generate_dict() log.info(particle_dict) #for efficiency turn the particle values list of dictionaries into a dictionary particle_values = {} for param in particle_dict.get('values'): particle_values[param['value_id']] = param['value'] # compare each key in the test to the data in the particle for key in test: log.info("key: %s", key) test_data = test[key] #get the correct data to compare to the test if key == 'internal_timestamp': particle_data = particle.get_value('internal_timestamp') #the timestamp is in the header part of the particle log.info("internal_timestamp %d", particle_data) else: particle_data = particle_values.get(key) #others are all part of the parsed values part of the particle if particle_data is None: #generally OK to ignore index keys in the test data, verify others log.warning("\nWarning: assert_result ignoring test key %s, does not exist in particle", key) else: log.info("test_data %s - particle_data %s", test_data, particle_data) if isinstance(test_data, float): # slightly different test for these values as they are floats. compare = numpy.abs(test_data - particle_data) <= 1e-5 self.assertTrue(compare) else: # otherwise they are all ints and should be exactly equal self.assertEqual(test_data, particle_data) def get_dict_from_yml(self, filename): """ This utility routine loads the contents of a yml file into a dictionary """ fid = open(os.path.join(RESOURCE_PATH, filename), 'r') result = yaml.load(fid) fid.close() return result def assert_state(self, in_process_data, unprocessed_data): self.assertEqual(self.parser._state[StateKey.IN_PROCESS_DATA], in_process_data) self.assertEqual(self.parser._state[StateKey.UNPROCESSED_DATA], unprocessed_data) self.assertEqual(self.state_callback_value[StateKey.IN_PROCESS_DATA], in_process_data) self.assertEqual(self.state_callback_value[StateKey.UNPROCESSED_DATA], unprocessed_data)
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, 'node58p3.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [] } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 3 particles particles = self.parser.get_records(3) # Make sure we obtained 3 particles self.assertTrue(len(particles) == 3) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes two chunks with one chunk having 5 samples with 3 returned and the other # chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 4 chunks. self.assert_state([[2818, 2982, 5, 3], [4059, 4673, 20, 0]], [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]]) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) # Make sure we obtained 22 particles self.assertTrue(len(particles) == 22) # Make sure the parser's new state indicates that there 3 remaining chunks in the UNPROCESSED_DATA, none of # which have WE samples. self.assert_state([], [[4058, 4059], [7423, 7424], [7594, 7623]]) # Create a new state to declare the UNPROCESSED_DATA including 4 chunks. Declare the IN_PROCESS_DATA as # the two chunks with WE samples one having 5 samples with 3 returned and the other having 20 samples with # 0 returned. new_state = {StateKey.UNPROCESSED_DATA: [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 3], [4059, 4673, 20, 0]]} # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes one chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 3 chunks within one corresponding to the chunk with WE samples. The # other two chunks contain invalid data. self.assert_state([[4059, 4673, 20, 0]], [[4058, 4673], [7423, 7424], [7594, 7623]]) # Close the file stream as we don't need it anymore stream_handle.close()
def _build_parser(self, parser_state, infile, data_key=None): """ Build and return the parser """ config = self._parser_config.get(data_key) # # If the key is WFP_ENG_STC_IMODEM, build the WFP parser. # if data_key == DataTypeKey.WFP_ENG_STC_IMODEM: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng__stc_imodem_particles', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'status_data_particle_class': WfpEngStcImodemStatusRecoveredDataParticle, 'start_data_particle_class': WfpEngStcImodemStartRecoveredDataParticle, 'engineering_data_particle_class': WfpEngStcImodemEngineeringRecoveredDataParticle } }) parser = WfpEngStcImodemParser( config, parser_state, infile, lambda state, ingested: self._save_parser_state( state, data_key, ingested), self._data_callback, self._sample_exception_callback) # # If the key is WFP_ENG_WFP_SIO_MULE, build the WFP SIO Mule parser. # elif data_key == DataTypeKey.WFP_ENG_WFP_SIO_MULE: config.update({ DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng_wfp_sio_mule', DataSetDriverConfigKeys.PARTICLE_CLASS: None, DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: { 'start_time_data_particle_class': WfpEngWfpSioMuleParserDataStartTimeParticle, 'status_data_particle_class': WfpEngWfpSioMuleParserDataStatusParticle, 'engineering_data_particle_class': WfpEngWfpSioMuleParserDataEngineeringParticle } }) parser = WfpEngWfpSioMuleParser( config, parser_state, infile, lambda state: self._save_parser_state(state, data_key), self._data_callback, self._sample_exception_callback) # # If the key is one that we're not expecting, don't build any parser. # else: raise ConfigurationException( "Invalid data_key supplied to build parser") return parser
def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, 'node58p3.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 3 particles particles = self.parser.get_records(3) # Make sure we obtained 3 particles self.assertTrue(len(particles) == 3) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes two chunks with one chunk having 5 samples with 3 returned and the other # chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 4 chunks. self.assert_state( [[2818, 2982, 5, 3], [4059, 4673, 20, 0]], [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]]) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) # Make sure we obtained 22 particles self.assertTrue(len(particles) == 22) # Make sure the parser's new state indicates that there 3 remaining chunks in the UNPROCESSED_DATA, none of # which have WE samples. self.assert_state([], [[4058, 4059], [7423, 7424], [7594, 7623]]) # Create a new state to declare the UNPROCESSED_DATA including 4 chunks. Declare the IN_PROCESS_DATA as # the two chunks with WE samples one having 5 samples with 3 returned and the other having 20 samples with # 0 returned. new_state = { StateKey.UNPROCESSED_DATA: [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 3], [4059, 4673, 20, 0]], StateKey.FILE_SIZE: stat_info.st_size } # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes one chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 3 chunks within one corresponding to the chunk with WE samples. The # other two chunks contain invalid data. self.assert_state([[4059, 4673, 20, 0]], [[4058, 4673], [7423, 7424], [7594, 7623]]) # Close the file stream as we don't need it anymore stream_handle.close()
class WfpEngWfpSioMuleParserUnitTestCase(ParserUnitTestCase): """ wfp_eng_wfp_sio_mule Parser unit test suite """ def state_callback(self, state): """ Call back method to watch what comes in via the position callback """ self.state_callback_value = state def pub_callback(self, pub): """ Call back method to watch what comes in via the publish callback """ self.publish_callback_value = pub def exception_callback(self, exception): """ Call back method to match what comes in via the exception callback """ self.exception_callback_value = exception def setUp(self): ParserUnitTestCase.setUp(self) self.config = { DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.wfp_eng_wfp_sio_mule', DataSetDriverConfigKeys.PARTICLE_CLASS: None } # Define test data particles and their associated timestamps which will be # compared with returned results self.file_ingested_value = None self.state_callback_value = None self.publish_callback_value = None self.test_eng_particle1 = { 'internal_timestamp': 3583889968.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_TIMESTAMP: 1374901168, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_CURRENT: 295.3860778808594, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_VOLTAGE: 10.81559944152832, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_PRESSURE: 2335.27001953125 } self.test_eng_particle2 = { 'internal_timestamp': 3583825326.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_TIMESTAMP: 1374836526, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_CURRENT: 0.0, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_VOLTAGE: 11.584400177001953, WfpEngWfpSioMuleParserDataEngineeringParticleKey.WFP_PROF_PRESSURE: 0.0 } self.test_start_particle1 = { 'internal_timestamp': 3583954925.0, WfpEngWfpSioMuleParserDataStartTimeParticleKey.CONTROLLER_TIMESTAMP: 1374973536, WfpEngWfpSioMuleParserDataStartTimeParticleKey.WFP_PROFILE_START: 1374966125, WfpEngWfpSioMuleParserDataStartTimeParticleKey.WFP_SENSOR_START: 1374966002 } self.test_status_particle1 = { 'internal_timestamp': 3583826263.0, WfpEngWfpSioMuleParserDataStatusParticleKey.CONTROLLER_TIMESTAMP: 1374837724, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_INDICATOR: -1, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_RAMP_STATUS: 0, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_PROFILE_STATUS: 14, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_PROFILE_STOP: 1374837463, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_SENSOR_STOP: 1374837591, WfpEngWfpSioMuleParserDataStatusParticleKey.WFP_DECIMATION_FACTOR: None } def test_simple(self): """ Read test data and pull out 5 data particles. Then examine each particle against its expected result. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 5 particles particles = self.parser.get_records(5) # Close the file stream as we don't need it anymore stream_handle.close() # Make sure we obtained 5 particles self.assertTrue(len(particles) == 5) # Obtain the expected 5 samples from a yml file test_data = self.get_dict_from_yml('first_five.yml') index = 0 for particle in particles: log.info(particle.generate_dict()) # Make sure each retrieved sample matches its expected result self.assert_result(test_data['data'][index], particles[index]) index += 1 def test_get_many(self): """ Read test data and pull out multiple data particles at one time. Assert that the results are those we expected. """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) log.info(len(particles)) # Make sure we obtained 30 particles self.assertTrue(len(particles) == 30) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_eng_particle1, particles[28]) # Attempt to retrieve 50 particles particles = self.parser.get_records(50) # Make sure we obtained 50 particles self.assertTrue(len(particles) == 50) for particle in particles: log.info(particle.generate_dict()) # Compare one of the particles to its expected result self.assert_result(self.test_start_particle1, particles[48]) stream_handle.close() def test_long_stream(self): """ Test a long stream """ file_path = os.path.join(RESOURCE_PATH, 'node58p1.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 200000 particles particles = self.parser.get_records(20000) # There is NOT 20000 in the file. Make sure we obtained 11456 particles. self.assertTrue(len(particles) == 11456) # Close the file stream as we don't need it anymore stream_handle.close() def test_mid_state_start(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as one WE chunk. Declare the IN_PROCESS_DATA as the one WE # chunk that has 5 samples with 1 of the samples as returned. initial_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 1]], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 1 particle self.assertTrue(len(particles) == 1) # Compare the particle to its expected result self.assert_result(self.test_status_particle1, particles[0]) stream_handle.close() def test_start_stop_set_state_and_resume(self): """ Test starting the parser in a state in the middle of processing """ file_path = os.path.join(RESOURCE_PATH, 'wfp_eng_wfp_sio_mule_small.DAT') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Create a new state to declare the UNPROCESSED_DATA as one chunk. Declare the IN_PROCESS_DATA as one chunk # that has 5 samples with 4 of the samples as returned. new_state = { StateKey.UNPROCESSED_DATA: [[2818, 2992]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 4]], StateKey.FILE_SIZE: stat_info.st_size } # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we only obtained 1 particle since the sample test file only holds one WE chunk with 5 total # samples with 4 already returned. self.assertTrue(len(particles) == 1) # Compare the retrieved particle to its expected result. self.assert_result(self.test_eng_particle2, particles[0]) # Attempt to retrieve 1 particle particles = self.parser.get_records(1) # Make sure we obtained 0 particles self.assertTrue(len(particles) == 0) # Close the file stream as we don't need it anymore stream_handle.close() def test_set_state(self): """ Test changing to a new state after initializing the parser and reading data, as if new data has been found and the state has changed """ file_path = os.path.join(RESOURCE_PATH, 'node58p3.dat') # Obtain statistics on the test sample file stat_info = os.stat(file_path) # Init the state to declare the UNPROCESSED_DATA as the full length of the file initial_state = { StateKey.UNPROCESSED_DATA: [[0, stat_info.st_size]], StateKey.IN_PROCESS_DATA: [], StateKey.FILE_SIZE: stat_info.st_size } # Open the file holding the test sample data stream_handle = open(file_path, 'rb') self.parser = WfpEngWfpSioMuleParser(self.config, initial_state, stream_handle, self.state_callback, self.pub_callback, self.exception_callback) # Attempt to retrieve 3 particles particles = self.parser.get_records(3) # Make sure we obtained 3 particles self.assertTrue(len(particles) == 3) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes two chunks with one chunk having 5 samples with 3 returned and the other # chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 4 chunks. self.assert_state( [[2818, 2982, 5, 3], [4059, 4673, 20, 0]], [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]]) # Attempt to retrieve 30 particles particles = self.parser.get_records(30) # Make sure we obtained 22 particles self.assertTrue(len(particles) == 22) # Make sure the parser's new state indicates that there 3 remaining chunks in the UNPROCESSED_DATA, none of # which have WE samples. self.assert_state([], [[4058, 4059], [7423, 7424], [7594, 7623]]) # Create a new state to declare the UNPROCESSED_DATA including 4 chunks. Declare the IN_PROCESS_DATA as # the two chunks with WE samples one having 5 samples with 3 returned and the other having 20 samples with # 0 returned. new_state = { StateKey.UNPROCESSED_DATA: [[2818, 2982], [4058, 4673], [7423, 7424], [7594, 7623]], StateKey.IN_PROCESS_DATA: [[2818, 2982, 5, 3], [4059, 4673, 20, 0]], StateKey.FILE_SIZE: stat_info.st_size } # Re-set the parser's state self.parser.set_state(new_state) # Attempt to retrieve 2 particles particles = self.parser.get_records(2) # Make sure we obtained 2 particles self.assertTrue(len(particles) == 2) # Make sure the state indicates: # 1) The IN_PROCESSED_DATA includes one chunk with 20 samples and 0 returned. # 2) The UNPROCESSED_DATA includes 3 chunks within one corresponding to the chunk with WE samples. The # other two chunks contain invalid data. self.assert_state([[4059, 4673, 20, 0]], [[4058, 4673], [7423, 7424], [7594, 7623]]) # Close the file stream as we don't need it anymore stream_handle.close() def assert_result(self, test, particle): """ Suite of tests to run against each returned particle and expected results of the same. The test parameter should be a dictionary that contains the keys to be tested in the particle the 'internal_timestamp' and 'position' keys are treated differently than others but can be verified if supplied """ particle_dict = particle.generate_dict() log.info(particle_dict) #for efficiency turn the particle values list of dictionaries into a dictionary particle_values = {} for param in particle_dict.get('values'): particle_values[param['value_id']] = param['value'] # compare each key in the test to the data in the particle for key in test: log.info("key: %s", key) test_data = test[key] #get the correct data to compare to the test if key == 'internal_timestamp': particle_data = particle.get_value('internal_timestamp') #the timestamp is in the header part of the particle log.info("internal_timestamp %d", particle_data) else: particle_data = particle_values.get(key) #others are all part of the parsed values part of the particle if particle_data is None: #generally OK to ignore index keys in the test data, verify others log.warning( "\nWarning: assert_result ignoring test key %s, does not exist in particle", key) else: log.info("test_data %s - particle_data %s", test_data, particle_data) if isinstance(test_data, float): # slightly different test for these values as they are floats. compare = numpy.abs(test_data - particle_data) <= 1e-5 self.assertTrue(compare) else: # otherwise they are all ints and should be exactly equal self.assertEqual(test_data, particle_data) def get_dict_from_yml(self, filename): """ This utility routine loads the contents of a yml file into a dictionary """ fid = open(os.path.join(RESOURCE_PATH, filename), 'r') result = yaml.load(fid) fid.close() return result def assert_state(self, in_process_data, unprocessed_data): self.assertEqual(self.parser._state[StateKey.IN_PROCESS_DATA], in_process_data) self.assertEqual(self.parser._state[StateKey.UNPROCESSED_DATA], unprocessed_data) self.assertEqual(self.state_callback_value[StateKey.IN_PROCESS_DATA], in_process_data) self.assertEqual(self.state_callback_value[StateKey.UNPROCESSED_DATA], unprocessed_data)