def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state'][ 'timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while (not done): if 'modified_state' in self.driver._driver_state[ '20130101.dosta.log']: log.debug( "Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get( 'modified_state')) done = True if not done: log.debug( "modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state']['timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while(not done): if 'modified_state' in self.driver._driver_state['20130101.dosta.log']: log.debug("Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get('modified_state' )) done = True if not done: log.debug("modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
def test_resume_file_start(self): """ Test the ability to restart the process """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log', DataSourceConfigKey.PARSER: None} self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log', DataSourceConfigKey.PARSER: None} self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log', DataSourceConfigKey.PARSER: {'position': 191, 'timestamp': 3590524819.861}} self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
from mi.idk.dataset.unit_test import DataSetQualificationTestCase from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys from mi.dataset.dataset_driver import DriverParameter from pyon.agent.agent import ResourceAgentState from interface.objects import ResourceAgentErrorEvent from mi.dataset.driver.issmcnsm.dosta.driver import IssmCnsmDOSTADDataSetDriver from mi.dataset.parser.issmcnsm_dostad import Issmcnsm_dostadParserDataParticle # Fill in driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.issmcnsm.dosta.driver', driver_class='IssmCnsmDOSTADDataSetDriver', agent_resource_id = '123xyz', agent_name = 'Agent007', agent_packet_config = IssmCnsmDOSTADDataSetDriver.stream_config(), startup_config = { 'harvester': { 'directory': '/tmp/dsatest', 'pattern': '*.dosta.log', 'frequency': 1, }, 'parser': {} } ) SAMPLE_STREAM = 'issmcnsm_dostad_parsed' ############################################################################### # INTEGRATION TESTS #
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state']['timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while(not done): if 'modified_state' in self.driver._driver_state['20130101.dosta.log']: log.debug("Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get('modified_state' )) done = True if not done: log.debug("modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state'][ 'timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while (not done): if 'modified_state' in self.driver._driver_state[ '20130101.dosta.log']: log.debug( "Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get( 'modified_state')) done = True if not done: log.debug( "modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
from mi.dataset.dataset_driver import DriverParameter from pyon.agent.agent import ResourceAgentState from interface.objects import ResourceAgentErrorEvent from mi.dataset.driver.issmcnsm.dosta.driver import IssmCnsmDOSTADDataSetDriver from mi.dataset.parser.issmcnsm_dostad import Issmcnsm_dostadParserDataParticle RESOURCE_ID = 'dostad' # Fill in driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.issmcnsm.dosta.driver', driver_class='IssmCnsmDOSTADDataSetDriver', agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=IssmCnsmDOSTADDataSetDriver.stream_config(), startup_config={ DataSourceConfigKey.RESOURCE_ID: RESOURCE_ID, DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.dosta.log', DataSetDriverConfigKeys.FREQUENCY: 1, DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 30, }, DataSourceConfigKey.PARSER: {} }) SAMPLE_STREAM = 'issmcnsm_dostad_parsed'