def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join( startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join( startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY), "20130102.flort.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file 20130101 is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = { '20130101.flort.log': { 'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, '20130102.flort.log': { 'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': { 'position': 146, 'timestamp': 3592854648.401 } } } # Create and store the new driver state, after completed reading 20130101.dosta.log self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.partial-result.yml', count=2, timeout=10)
def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY) file_path = os.path.join(directory, "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log self.memento = {'20130101.flort.log':{'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.flort.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.flort.log', "20130101.flort.log") to = gevent.Timeout(30) to.start() done = False try: while(not done): if 'modified_state' in self.driver._driver_state['20130101.flort.log']: log.debug("Found modified state %s", self.driver._driver_state['20130101.flort.log'].get('modified_state' )) done = True if not done: log.debug("modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "20130102.flort.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file 20130101 is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = {'20130101.flort.log':{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, '20130102.flort.log':{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 146, 'timestamp': 3592854648.401} } } # Create and store the new driver state, after completed reading 20130101.dosta.log self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.partial-result.yml', count=2, timeout=10)
def test_resume_file_start(self): """ Test the ability to restart the process """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.flort.log', DataSourceConfigKey.PARSER: None} self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)
from mi.idk.dataset.unit_test import DataSetQualificationTestCase from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys from mi.dataset.dataset_driver import DriverParameter from pyon.agent.agent import ResourceAgentState from mi.dataset.driver.issmcnsm.flort.driver import IssmCnsmFLORTDDataSetDriver from mi.dataset.parser.issmcnsm_flortd import Issmcnsm_flortdParserDataParticle # Fill in driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.issmcnsm.flort.driver', driver_class='IssmCnsmFLORTDDataSetDriver', agent_resource_id = '123xyz', agent_name = 'Agent007', agent_packet_config = IssmCnsmFLORTDDataSetDriver.stream_config(), startup_config = { DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.flort.log', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} } ) SAMPLE_STREAM = 'issmcnsm_flortd_parsed' ###############################################################################
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_1.txt.result.yml', count=2, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.flort.log', "20130104.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=15, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130105.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=2, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] file_path = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = {'20130101.flort.log':{'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY), "20130102.flort.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file 20130101 is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = {'20130101.flort.log':{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, '20130102.flort.log':{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 146, 'timestamp': 3592854648.401} } } # Create and store the new driver state, after completed reading 20130101.dosta.log self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.partial-result.yml', count=2, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY) file_path = os.path.join(directory, "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log self.memento = {'20130101.flort.log':{'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.flort.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.flort.log', "20130101.flort.log") to = gevent.Timeout(30) to.start() done = False try: while(not done): if 'modified_state' in self.driver._driver_state['20130101.flort.log']: log.debug("Found modified state %s", self.driver._driver_state['20130101.flort.log'].get('modified_state' )) done = True if not done: log.debug("modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
from mi.idk.dataset.unit_test import DataSetQualificationTestCase from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys from mi.dataset.dataset_driver import DriverParameter from pyon.agent.agent import ResourceAgentState from mi.dataset.driver.issmcnsm.flort.driver import IssmCnsmFLORTDDataSetDriver from mi.dataset.parser.issmcnsm_flortd import Issmcnsm_flortdParserDataParticle # Fill in driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.issmcnsm.flort.driver', driver_class='IssmCnsmFLORTDDataSetDriver', agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=IssmCnsmFLORTDDataSetDriver.stream_config(), startup_config={ DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.flort.log', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} }) SAMPLE_STREAM = 'issmcnsm_flortd_parsed' ############################################################################### # INTEGRATION TESTS #
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_1.txt.result.yml', count=2, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.flort.log', "20130104.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=15, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130105.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=2, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] file_path = os.path.join( startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = { '20130101.flort.log': { 'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") startup_config = self._driver_config()['startup_config'] file_path_1 = os.path.join( startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY), "20130101.flort.log") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join( startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY), "20130102.flort.log") # need to reset file mod time since file is created again mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log # Note, since file 20130101 is ingested, parser state is not looked at, in a real run there would be a state in there self.memento = { '20130101.flort.log': { 'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, '20130102.flort.log': { 'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': { 'position': 146, 'timestamp': 3592854648.401 } } } # Create and store the new driver state, after completed reading 20130101.dosta.log self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.partial-result.yml', count=2, timeout=10) def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY) file_path = os.path.join(directory, "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log self.memento = { '20130101.flort.log': { 'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.flort.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.flort.log', "20130101.flort.log") to = gevent.Timeout(30) to.start() done = False try: while (not done): if 'modified_state' in self.driver._driver_state[ '20130101.flort.log']: log.debug( "Found modified state %s", self.driver._driver_state['20130101.flort.log'].get( 'modified_state')) done = True if not done: log.debug( "modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
def test_modified(self): """ Test for detection of an ingested file that has been modifed after ingestion """ self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY) file_path = os.path.join(directory, "20130101.flort.log") # need to reset file mod time since file is created again mod_time = os.path.getmtime(file_path) file_size = os.path.getsize(file_path) with open(file_path) as filehandle: md5_checksum = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state, after completed reading 20130101.flort.log self.memento = { '20130101.flort.log': { 'ingested': True, 'file_mod_date': mod_time, 'file_checksum': md5_checksum, 'file_size': file_size, 'parser_state': {} } } self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # overwrite the old 20130101.flort.log file # NOTE: this does not make you wait until file mod time, since it copies the original file # modification time, not when you copy the file in running this test self.create_sample_data('test_data_2.flort.log', "20130101.flort.log") to = gevent.Timeout(30) to.start() done = False try: while (not done): if 'modified_state' in self.driver._driver_state[ '20130101.flort.log']: log.debug( "Found modified state %s", self.driver._driver_state['20130101.flort.log'].get( 'modified_state')) done = True if not done: log.debug( "modification not detected yet, sleep some more...") gevent.sleep(5) except Timeout: log.error("Failed to find modified file after ingestion") self.fail("Failed to find modified file after ingestion") finally: to.cancel()
class IntegrationTest(DataSetIntegrationTestCase): def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_1.txt.result.yml', count=2, timeout=10) self.clear_async_data() self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) self.clear_async_data() # skipping a file index 20130103 here to make sure it still finds the new file self.create_sample_data('test_data_3.flort.log', "20130104.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=15, timeout=30) self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130105.flort.log") self.assert_data(Issmcnsm_flortdParserDataParticle, count=2, timeout=10) def test_resume_file_start(self): """ Test the ability to restart the process """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.flort.log', DataSourceConfigKey.PARSER: None} self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10) def test_resume_mid_file(self): """ Test the ability to restart the process in the middle of a file """ # Create and store the new driver state, after completed reading 20130101.dosta.log self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.flort.log', DataSourceConfigKey.PARSER: {'position': 146, 'timestamp': 3592854648.401}} self.driver = IssmCnsmFLORTDDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.flort.log', "20130101.flort.log") self.create_sample_data('test_data_2.flort.log', "20130102.flort.log") self.driver.start_sampling() # verify data is produced self.assert_data(Issmcnsm_flortdParserDataParticle, 'test_data_2.txt.partial-result.yml', count=2, timeout=10)