class IntegrationTest(DataSetIntegrationTestCase): def clean_file(self): # remove just the file we are using driver_config = self._driver_config()['startup_config'] log.debug('startup config %s', driver_config) fullfile = os.path.join(driver_config['harvester']['directory'], driver_config['harvester']['pattern']) if os.path.exists(fullfile): os.remove(fullfile) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clean_file() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data("node59p1_step4.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, count=4, timeout=10) self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, count=1, timeout=10) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ self.clean_file() # create the file so that it is unreadable self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(IOError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() # Create and store the new driver state self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 314, 'last_checksum': '515e5da08a6b4bb0d197e62c410da532'}, DataSourceConfigKey.PARSER: {'in_process_data': [], 'unprocessed_data':[[0,69]], 'timestamp': 3583581301.0}} self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) def test_sequences(self): """ Test new sequence flags are set correctly """ self.clean_file() self.driver.start_sampling() self.clear_async_data() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests (no new sequences) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml', count=2, timeout=10) # This file has had a section of FL data replaced with 0s, this should start a new # sequence for the data following the missing AD data self.clear_async_data() self.create_sample_data('node59p1_step3.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml', count=3, timeout=10) # Now fill in the zeroed section from step3, this should just return the new # data with a new sequence flag self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml', count=1, timeout=10) # start over now, using step 4, make sure sequence flags just account for # missing data in file (there are some sections of bad data that don't # match in headers self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml', count=6, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def clean_file(self): # remove just the file we are using driver_config = self._driver_config()['startup_config'] log.debug('startup config %s', driver_config) fullfile = os.path.join(driver_config['harvester']['directory'], driver_config['harvester']['pattern']) if os.path.exists(fullfile): os.remove(fullfile) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clean_file() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data("node59p1_step4.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, count=4, timeout=10) self.driver.stop_sampling() # Reset the driver with no memento self.memento = None self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, count=1, timeout=10) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ self.clean_file() # create the file so that it is unreadable self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(ValueError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") driver_config = self._driver_config()['startup_config'] fullfile = os.path.join(driver_config['harvester']['directory'], driver_config['harvester']['pattern']) mod_time = os.path.getmtime(fullfile) # Create and store the new driver state self.memento = {DriverStateKey.FILE_SIZE: 314, DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532', DriverStateKey.FILE_MOD_DATE: mod_time, DriverStateKey.PARSER_STATE: {'in_process_data': [], 'unprocessed_data':[[0,69]], 'timestamp': 3583581301.0} } self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) def test_sequences(self): """ Test new sequence flags are set correctly """ self.clean_file() self.driver.start_sampling() self.clear_async_data() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests (no new sequences) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml', count=2, timeout=10) # This file has had a section of FL data replaced with 0s, this should start a new # sequence for the data following the missing AD data self.clear_async_data() self.create_sample_data('node59p1_step3.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml', count=3, timeout=10) # Now fill in the zeroed section from step3, this should just return the new # data with a new sequence flag self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml', count=1, timeout=10) # start over now, using step 4, make sure sequence flags just account for # missing data in file (there are some sections of bad data that don't # match in headers self.driver.stop_sampling() # Reset the driver with no memento self.memento = None self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) self.driver.start_sampling() self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml', count=6, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ # Start sampling self.driver.start_sampling() self.clear_async_data() self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1.txt.result.yml', count=2, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data_set_dir("node59p1_step4.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, count=4) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ # create the file so that it is unreadable self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", mode=000, copy_metadata=False) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(ValueError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) driver_config = self._driver_config()['startup_config'] dosta_telem_config = driver_config['harvester'][DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED] fullfile = os.path.join(dosta_telem_config['directory'], dosta_telem_config['pattern']) mod_time = os.path.getmtime(fullfile) # Create and store the new driver state self.memento = { DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: { "node59p1.dat": { DriverStateKey.FILE_SIZE: 314, DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532', DriverStateKey.FILE_MOD_DATE: mod_time, DriverStateKey.PARSER_STATE: { 'in_process_data': [], 'unprocessed_data':[[0,69]] } } } } self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.driver.start_sampling() # verify data is produced self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1) def test_back_fill(self): """ Test new sequence flags are set correctly """ self.driver.start_sampling() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) # This file has had a section of DO data replaced with 0s self.clear_async_data() self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml', count=3) # Now fill in the zeroed section from step3, this should just return the new # data self.clear_async_data() self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml', count=1) # start over now, using step 4 self.driver.stop_sampling() # Reset the driver with no memento self.memento = None self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) self.driver.start_sampling() self.clear_async_data() self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-4.txt.result.yml', count=7) def test_all_good(self): """ Test that a set of data with no bad data, where there is no remaining unprocessed data in between """ self.driver.start_sampling() self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat") self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml', count=1) def test_all_good(self): """ Test that a set of data with no bad data, where there is no remaining unprocessed data in between """ self.driver.start_sampling() self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat") self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml', count=1)
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ # Start sampling self.driver.start_sampling() self.clear_async_data() self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data( (DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1.txt.result.yml', count=2, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data_set_dir("node59p1_step4.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, count=4) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ # create the file so that it is unreadable self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", mode=000, copy_metadata=False) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(ValueError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) driver_config = self._driver_config()['startup_config'] dosta_telem_config = driver_config['harvester'][ DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED] fullfile = os.path.join(dosta_telem_config['directory'], dosta_telem_config['pattern']) mod_time = os.path.getmtime(fullfile) # Create and store the new driver state self.memento = { DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: { "node59p1.dat": { DriverStateKey.FILE_SIZE: 314, DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532', DriverStateKey.FILE_MOD_DATE: mod_time, DriverStateKey.PARSER_STATE: { StateKey.IN_PROCESS_DATA: [], StateKey.UNPROCESSED_DATA: [[0, 69]], StateKey.FILE_SIZE: 314 } } } } self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.driver.start_sampling() # verify data is produced self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml', count=1) def test_back_fill(self): """ Test new sequence flags are set correctly """ self.driver.start_sampling() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests self.clear_async_data() self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data( (DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) # This file has had a section of DO data replaced with 0s self.clear_async_data() self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml', count=3) # Now fill in the zeroed section from step3, this should just return the new # data self.clear_async_data() self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml', count=1) # start over now, using step 4 self.driver.stop_sampling() # Reset the driver with no memento self.memento = None self.driver = MflmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) self.driver.start_sampling() self.clear_async_data() self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat", copy_metadata=False) self.assert_data( (DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-4.txt.result.yml', count=7) def test_all_good(self): """ Test that a set of data with no bad data, where there is no remaining unprocessed data in between """ self.driver.start_sampling() self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat") self.assert_data( (DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml', count=1) def test_all_good(self): """ Test that a set of data with no bad data, where there is no remaining unprocessed data in between """ self.driver.start_sampling() self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat") self.assert_data( (DostadParserDataParticle, DostadMetadataDataParticle), 'test_data_1-2.txt.result.yml', count=3) self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat") self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml', count=1)