class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state']['timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while(not done): if 'modified_state' in self.driver._driver_state['20130101.dosta.log']: log.debug("Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get('modified_state' )) done = True if not done: log.debug("modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log', DataSourceConfigKey.PARSER: None} self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log', DataSourceConfigKey.PARSER: {'position': 191, 'timestamp': 3590524819.861}} self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log") self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(path_1, True, 190), '20130102.dosta.log': self.get_file_state(path_2, False, 191) } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.memento['20130102.dosta.log']['parser_state'][ 'timestamp'] = 3590524819.861 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log") self.memento = { '20130101.dosta.log': self.get_file_state(file_path, True, 190), } self.memento['20130101.dosta.log']['parser_state'][ 'timestamp'] = 3590524817.862 self.driver = IssmCnsmDOSTADDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.dosta.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log") to = gevent.Timeout(30) to.start() done = False try: while (not done): if 'modified_state' in self.driver._driver_state[ '20130101.dosta.log']: log.debug( "Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get( 'modified_state')) done = True if not done: log.debug( "modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()