Ejemplo n.º 1
0
    def test_resume_mid_file(self):
        """
        Test the ability to restart the process in the middle of a file
        """
        path_1 = self.create_sample_data('test_data_1.dosta.log',
                                         "20130101.dosta.log")
        path_2 = self.create_sample_data('test_data_2.dosta.log',
                                         "20130102.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(path_1, True, 190),
            '20130102.dosta.log': self.get_file_state(path_2, False, 191)
        }
        self.memento['20130101.dosta.log']['parser_state'][
            'timestamp'] = 3590524817.862
        self.memento['20130102.dosta.log']['parser_state'][
            'timestamp'] = 3590524819.861

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         'test_data_2.txt.partial-result.yml',
                         count=3,
                         timeout=10)
Ejemplo n.º 2
0
    def test_modified(self):
        """
        Test for detection of an ingested file that has been modifed after ingestion
        """
        file_path = self.create_sample_data('test_data_1.dosta.log',
                                            "20130101.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state'][
            'timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # overwrite the old 20130101.dosta.log file
        # NOTE: this does not make you wait until file mod time, since it copies the original file
        # modification time, not when you copy the file in running this test
        self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log")

        to = gevent.Timeout(30)
        to.start()
        done = False
        try:
            while (not done):
                if 'modified_state' in self.driver._driver_state[
                        '20130101.dosta.log']:
                    log.debug(
                        "Found modified state %s",
                        self.driver._driver_state['20130101.dosta.log'].get(
                            'modified_state'))
                    done = True

                if not done:
                    log.debug(
                        "modification not detected yet, sleep some more...")
                    gevent.sleep(5)
        except Timeout:
            log.error("Failed to find modified file after ingestion")
            self.fail("Failed to find modified file after ingestion")
        finally:
            to.cancel()
Ejemplo n.º 3
0
    def test_resume_mid_file(self):
        """
        Test the ability to restart the process in the middle of a file
        """
        path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(path_1, True, 190),
            '20130102.dosta.log': self.get_file_state(path_2, False, 191)
        }
        self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862
        self.memento['20130102.dosta.log']['parser_state']['timestamp'] = 3590524819.861

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
Ejemplo n.º 4
0
    def test_modified(self):
        """
        Test for detection of an ingested file that has been modifed after ingestion
        """
        file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # overwrite the old 20130101.dosta.log file
        # NOTE: this does not make you wait until file mod time, since it copies the original file
        # modification time, not when you copy the file in running this test
        self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log")

        to = gevent.Timeout(30)
        to.start()
        done = False
        try:
            while(not done):
                if 'modified_state' in self.driver._driver_state['20130101.dosta.log']:
                    log.debug("Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get('modified_state' ))
                    done = True

                if not done:
                    log.debug("modification not detected yet, sleep some more...")
                    gevent.sleep(5)
        except Timeout:
            log.error("Failed to find modified file after ingestion")
            self.fail("Failed to find modified file after ingestion")
        finally:
            to.cancel()
Ejemplo n.º 5
0
    def test_resume_file_start(self):
        """
        Test the ability to restart the process
        """
        # Create and store the new driver state, after completed reading  20130101.dosta.log
        self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log',
                        DataSourceConfigKey.PARSER: None}
        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)
Ejemplo n.º 6
0
class IntegrationTest(DataSetIntegrationTestCase):

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        self.clear_sample_data()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)

        self.clear_async_data()
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)

        self.clear_async_data()
        # skipping a file index 20130103 here to make sure it still finds the new file
        self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30)

        self.driver.stop_sampling()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10)

    def test_resume_file_start(self):
        """
        Test the ability to restart the process
        """
        # Create and store the new driver state, after completed reading  20130101.dosta.log
        self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log',
                        DataSourceConfigKey.PARSER: None}
        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)

    def test_resume_mid_file(self):
        """
        Test the ability to restart the process in the middle of a file
        """
        # Create and store the new driver state, after completed reading  20130101.dosta.log
        self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/20130101.dosta.log',
                        DataSourceConfigKey.PARSER: {'position': 191, 'timestamp': 3590524819.861}}
        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)
Ejemplo n.º 7
0
from mi.idk.dataset.unit_test import DataSetQualificationTestCase
from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys
from mi.dataset.dataset_driver import DriverParameter
from pyon.agent.agent import ResourceAgentState
from interface.objects import ResourceAgentErrorEvent

from mi.dataset.driver.issmcnsm.dosta.driver import IssmCnsmDOSTADDataSetDriver
from mi.dataset.parser.issmcnsm_dostad import Issmcnsm_dostadParserDataParticle

# Fill in driver details
DataSetTestCase.initialize(
    driver_module='mi.dataset.driver.issmcnsm.dosta.driver',
    driver_class='IssmCnsmDOSTADDataSetDriver',
    agent_resource_id = '123xyz',
    agent_name = 'Agent007',
    agent_packet_config = IssmCnsmDOSTADDataSetDriver.stream_config(),
    startup_config = {
        'harvester':
        {
            'directory': '/tmp/dsatest',
            'pattern': '*.dosta.log',
            'frequency': 1,
        },
        'parser': {}
    }
)

SAMPLE_STREAM = 'issmcnsm_dostad_parsed'

###############################################################################
#                            INTEGRATION TESTS                                #
Ejemplo n.º 8
0
class IntegrationTest(DataSetIntegrationTestCase):

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        self.clear_sample_data()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)

        self.clear_async_data()
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)

        self.clear_async_data()
        # skipping a file index 20130103 here to make sure it still finds the new file
        self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, count=36, timeout=30)

        self.driver.stop_sampling()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle, count=1, timeout=10)

    def test_resume_file_start(self):
        """
        Test the ability to restart the process
        """
        file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.result.yml', count=4, timeout=10)

    def test_resume_mid_file(self):
        """
        Test the ability to restart the process in the middle of a file
        """
        path_1 = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        path_2 = self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(path_1, True, 190),
            '20130102.dosta.log': self.get_file_state(path_2, False, 191)
        }
        self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862
        self.memento['20130102.dosta.log']['parser_state']['timestamp'] = 3590524819.861

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle, 'test_data_2.txt.partial-result.yml', count=3, timeout=10)

    def test_modified(self):
        """
        Test for detection of an ingested file that has been modifed after ingestion
        """
        file_path = self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state']['timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # overwrite the old 20130101.dosta.log file
        # NOTE: this does not make you wait until file mod time, since it copies the original file
        # modification time, not when you copy the file in running this test
        self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log")

        to = gevent.Timeout(30)
        to.start()
        done = False
        try:
            while(not done):
                if 'modified_state' in self.driver._driver_state['20130101.dosta.log']:
                    log.debug("Found modified state %s", self.driver._driver_state['20130101.dosta.log'].get('modified_state' ))
                    done = True

                if not done:
                    log.debug("modification not detected yet, sleep some more...")
                    gevent.sleep(5)
        except Timeout:
            log.error("Failed to find modified file after ingestion")
            self.fail("Failed to find modified file after ingestion")
        finally:
            to.cancel()
Ejemplo n.º 9
0
class IntegrationTest(DataSetIntegrationTestCase):
    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        self.clear_sample_data()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130101.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         'test_data_1.txt.result.yml',
                         count=1,
                         timeout=10)

        self.clear_async_data()
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         'test_data_2.txt.result.yml',
                         count=4,
                         timeout=10)

        self.clear_async_data()
        # skipping a file index 20130103 here to make sure it still finds the new file
        self.create_sample_data('test_data_3.dosta.log', "20130104.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         count=36,
                         timeout=30)

        self.driver.stop_sampling()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.dosta.log', "20130105.dosta.log")
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         count=1,
                         timeout=10)

    def test_resume_file_start(self):
        """
        Test the ability to restart the process
        """
        file_path = self.create_sample_data('test_data_1.dosta.log',
                                            "20130101.dosta.log")
        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state'][
            'timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data('test_data_2.dosta.log', "20130102.dosta.log")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         'test_data_2.txt.result.yml',
                         count=4,
                         timeout=10)

    def test_resume_mid_file(self):
        """
        Test the ability to restart the process in the middle of a file
        """
        path_1 = self.create_sample_data('test_data_1.dosta.log',
                                         "20130101.dosta.log")
        path_2 = self.create_sample_data('test_data_2.dosta.log',
                                         "20130102.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(path_1, True, 190),
            '20130102.dosta.log': self.get_file_state(path_2, False, 191)
        }
        self.memento['20130101.dosta.log']['parser_state'][
            'timestamp'] = 3590524817.862
        self.memento['20130102.dosta.log']['parser_state'][
            'timestamp'] = 3590524819.861

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(Issmcnsm_dostadParserDataParticle,
                         'test_data_2.txt.partial-result.yml',
                         count=3,
                         timeout=10)

    def test_modified(self):
        """
        Test for detection of an ingested file that has been modifed after ingestion
        """
        file_path = self.create_sample_data('test_data_1.dosta.log',
                                            "20130101.dosta.log")

        self.memento = {
            '20130101.dosta.log': self.get_file_state(file_path, True, 190),
        }
        self.memento['20130101.dosta.log']['parser_state'][
            'timestamp'] = 3590524817.862

        self.driver = IssmCnsmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # overwrite the old 20130101.dosta.log file
        # NOTE: this does not make you wait until file mod time, since it copies the original file
        # modification time, not when you copy the file in running this test
        self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log")

        to = gevent.Timeout(30)
        to.start()
        done = False
        try:
            while (not done):
                if 'modified_state' in self.driver._driver_state[
                        '20130101.dosta.log']:
                    log.debug(
                        "Found modified state %s",
                        self.driver._driver_state['20130101.dosta.log'].get(
                            'modified_state'))
                    done = True

                if not done:
                    log.debug(
                        "modification not detected yet, sleep some more...")
                    gevent.sleep(5)
        except Timeout:
            log.error("Failed to find modified file after ingestion")
            self.fail("Failed to find modified file after ingestion")
        finally:
            to.cancel()
Ejemplo n.º 10
0
from mi.dataset.dataset_driver import DriverParameter
from pyon.agent.agent import ResourceAgentState
from interface.objects import ResourceAgentErrorEvent

from mi.dataset.driver.issmcnsm.dosta.driver import IssmCnsmDOSTADDataSetDriver
from mi.dataset.parser.issmcnsm_dostad import Issmcnsm_dostadParserDataParticle

RESOURCE_ID = 'dostad'

# Fill in driver details
DataSetTestCase.initialize(
    driver_module='mi.dataset.driver.issmcnsm.dosta.driver',
    driver_class='IssmCnsmDOSTADDataSetDriver',
    agent_resource_id='123xyz',
    agent_name='Agent007',
    agent_packet_config=IssmCnsmDOSTADDataSetDriver.stream_config(),
    startup_config={
        DataSourceConfigKey.RESOURCE_ID: RESOURCE_ID,
        DataSourceConfigKey.HARVESTER: {
            DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
            DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
            DataSetDriverConfigKeys.PATTERN: '*.dosta.log',
            DataSetDriverConfigKeys.FREQUENCY: 1,
            DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 30,
        },
        DataSourceConfigKey.PARSER: {}
    })

SAMPLE_STREAM = 'issmcnsm_dostad_parsed'