def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = IssmRiNUTNRBDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback)
def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.create_sample_data('test_data_1.log', "DATA001.log") self.create_sample_data('test_data_3.log', "DATA002.log") # get file metadata for use in state memento startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "DATA001.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "DATA002.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading "DATA001.log" # Note, since file "DATA001.log" is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = {"DATA001.log":{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.log":{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 2628} } } self.driver = IssmRiNUTNRBDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(NutnrbDataParticle, 'test_data_3.log.partial_results.yml', count=5, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def setUp(self): super(IntegrationTest, self).setUp() def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = IssmRiNUTNRBDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped. """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.log', "DATA001.log") self.assert_data(NutnrbDataParticle, 'test_data_1.log.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_3.log', "DATA002.log") self.assert_data(NutnrbDataParticle, 'test_data_3.log.result.yml', count=8, timeout=10) self.clear_async_data() self.create_sample_data('20121213.nutnr.log', "DATA003.log") self.assert_data(NutnrbDataParticle, count=435, timeout=20) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.log', "DATA004.log") self.assert_data(NutnrbDataParticle, count=1, timeout=10) def test_stop_resume(self): """ Test the ability to stop and restart the process """ self.create_sample_data('test_data_1.log', "DATA001.log") self.create_sample_data('test_data_3.log', "DATA002.log") # get file metadata for use in state memento startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "DATA001.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "DATA002.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading "DATA001.log" # Note, since file "DATA001.log" is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = {"DATA001.log":{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.log":{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 2628} } } self.driver = IssmRiNUTNRBDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(NutnrbDataParticle, 'test_data_3.log.partial_results.yml', count=5, timeout=10)
from mi.core.instrument.instrument_driver import DriverEvent from mi.dataset.parser.nutnrb import NutnrbDataParticle from mi.dataset.driver.issm.nutnrb.driver import IssmRiNUTNRBDataSetDriver from pyon.agent.agent import ResourceAgentState from interface.objects import CapabilityType from interface.objects import AgentCapability from interface.objects import ResourceAgentErrorEvent from interface.objects import ResourceAgentConnectionLostErrorEvent DataSetTestCase.initialize( driver_module='mi.dataset.driver.issm.nutnrb.driver', driver_class='IssmRiNUTNRBDataSetDriver', agent_resource_id = '123xyz', agent_name = 'Agent007', agent_packet_config = IssmRiNUTNRBDataSetDriver.stream_config(), startup_config = { DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.log', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} } ) SAMPLE_STREAM = 'nutnrb_parsed' ###############################################################################