def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() # Create and store the new driver state self.memento = { DataSourceConfigKey.HARVESTER: { 'last_filesize': 1300, 'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93' }, DataSourceConfigKey.PARSER: { 'in_process_data': [], 'unprocessed_data': [[0, 32], [222, 871], [1257, 1300]], 'timestamp': 3583725976.97 } } self.driver = MflmADCPSDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10)
def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() # Create and store the new driver state self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 1300, 'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'}, DataSourceConfigKey.PARSER: {'in_process_data': [], 'unprocessed_data':[[0,32], [222,871], [1257,1300]], 'timestamp': 3583725976.97}} self.driver = MflmADCPSDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10)
from mi.idk.dataset.unit_test import DataSetQualificationTestCase from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys from mi.dataset.dataset_driver import DriverParameter, DriverStateKey from mi.dataset.driver.mflm.adcp.driver import MflmADCPSDataSetDriver, DataSourceKey from mi.dataset.parser.adcps import AdcpsParserDataParticle TELEM_DIR = '/tmp/dsatest' # Fill in the blanks to initialize data set test case DataSetTestCase.initialize( driver_module='mi.dataset.driver.mflm.adcp.driver', driver_class='MflmADCPSDataSetDriver', agent_resource_id = '123xyz', agent_name = 'Agent007', agent_packet_config = MflmADCPSDataSetDriver.stream_config(), startup_config = { DataSourceConfigKey.RESOURCE_ID: 'adcps', DataSourceConfigKey.HARVESTER: { DataSourceKey.ADCPS_JLN_SIO_MULE: { DataSetDriverConfigKeys.DIRECTORY: TELEM_DIR, DataSetDriverConfigKeys.PATTERN: 'node59p1.dat', DataSetDriverConfigKeys.FREQUENCY: 1, DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 5 } }, DataSourceConfigKey.PARSER: { DataSourceKey.ADCPS_JLN_SIO_MULE: {} } } )
from mi.dataset.dataset_driver import DriverParameter, DriverStateKey from mi.dataset.driver.mflm.adcp.driver import MflmADCPSDataSetDriver from mi.dataset.parser.adcps import AdcpsParserDataParticle from pyon.agent.agent import ResourceAgentState from interface.objects import ResourceAgentErrorEvent from interface.objects import ResourceAgentConnectionLostErrorEvent # Fill in the blanks to initialize data set test case DataSetTestCase.initialize( driver_module='mi.dataset.driver.mflm.adcp.driver', driver_class='MflmADCPSDataSetDriver', agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=MflmADCPSDataSetDriver.stream_config(), startup_config={ DataSourceConfigKey.RESOURCE_ID: 'adcps', DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: 'node59p1.dat', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} }) SAMPLE_STREAM = 'adcps_parsed' ###############################################################################
class IntegrationTest(DataSetIntegrationTestCase): def clean_file(self): # remove just the file we are using driver_config = self._driver_config()['startup_config'] log.debug('startup config %s', driver_config) fullfile = os.path.join(driver_config['harvester']['directory'], driver_config['harvester']['pattern']) if os.path.exists(fullfile): os.remove(fullfile) def test_get(self): self.clean_file() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data("node59p1_step4.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, count=2, timeout=10) self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, count=1, timeout=10) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ self.clean_file() # create the file so that it is unreadable self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(IOError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() # Create and store the new driver state self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 1300, 'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'}, DataSourceConfigKey.PARSER: {'in_process_data': [], 'unprocessed_data':[[0,32], [222,871], [1257,1300]], 'timestamp': 3583725976.97}} self.driver = MflmADCPSDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) def test_sequences(self): """ Test new sequence flags are set correctly. There is only one file that just has data appended or inserted into it, so new sequences can occur in both cases, or if there is missing data in between two sequences """ self.clean_file() self.driver.start_sampling() self.clear_async_data() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests (no new sequences) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1-2.txt.result.yml', count=2, timeout=10) # This file has had a section of AD data replaced with 0s, this should start a new # sequence for the data following the missing AD data self.clear_async_data() self.create_sample_data('node59p1_step3.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_3.txt.result.yml', count=1, timeout=10) # Now fill in the zeroed section from step3, this should just return the new # data with a new sequence flag self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_4.txt.result.yml', count=1, timeout=10) # start over now, using step 4, make sure sequence flags just account for # missing data in file (there are some sections of bad data that don't # match in headers, [0-32], [222-871], [1833-2000] self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1-4.txt.result.yml', count=4, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def clean_file(self): # remove just the file we are using driver_config = self._driver_config()['startup_config'] log.debug('startup config %s', driver_config) fullfile = os.path.join(driver_config['harvester']['directory'], driver_config['harvester']['pattern']) if os.path.exists(fullfile): os.remove(fullfile) def test_get(self): self.clean_file() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) # there is only one file we read from, this example 'appends' data to # the end of the node59p1.dat file, and the data from the new append # is returned (not including the original data from _step1) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) # now 'appends' the rest of the data and just check if we get the right number self.clear_async_data() self.create_sample_data("node59p1_step4.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, count=2, timeout=10) self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data("node59p1_step1.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, count=1, timeout=10) def test_harvester_new_file_exception(self): """ Test an exception raised after the driver is started during the file read. Should call the exception callback. """ self.clean_file() # create the file so that it is unreadable self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000) # Start sampling and watch for an exception self.driver.start_sampling() self.assert_exception(IOError) # At this point the harvester thread is dead. The agent # exception handle should handle this case. def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.clean_file() # Create and store the new driver state self.memento = { DataSourceConfigKey.HARVESTER: { 'last_filesize': 1300, 'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93' }, DataSourceConfigKey.PARSER: { 'in_process_data': [], 'unprocessed_data': [[0, 32], [222, 871], [1257, 1300]], 'timestamp': 3583725976.97 } } self.driver = MflmADCPSDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.driver.start_sampling() # verify data is produced self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml', count=1, timeout=10) def test_sequences(self): """ Test new sequence flags are set correctly. There is only one file that just has data appended or inserted into it, so new sequences can occur in both cases, or if there is missing data in between two sequences """ self.clean_file() self.driver.start_sampling() self.clear_async_data() # step 2 contains 2 blocks, start with this and get both since we used them # separately in other tests (no new sequences) self.clear_async_data() self.create_sample_data("node59p1_step2.dat", "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1-2.txt.result.yml', count=2, timeout=10) # This file has had a section of AD data replaced with 0s, this should start a new # sequence for the data following the missing AD data self.clear_async_data() self.create_sample_data('node59p1_step3.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_3.txt.result.yml', count=1, timeout=10) # Now fill in the zeroed section from step3, this should just return the new # data with a new sequence flag self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_4.txt.result.yml', count=1, timeout=10) # start over now, using step 4, make sure sequence flags just account for # missing data in file (there are some sections of bad data that don't # match in headers, [0-32], [222-871], [1833-2000] self.driver.stop_sampling() # reset the parser and harvester states self.driver.clear_states() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('node59p1_step4.dat', "node59p1.dat") self.assert_data(AdcpsParserDataParticle, 'test_data_1-4.txt.result.yml', count=4, timeout=10)