예제 #1
0
class IntegrationTest(DataSetIntegrationTestCase):

    def clean_file(self):
        # remove just the file we are using
        driver_config = self._driver_config()['startup_config']
        log.debug('startup config %s', driver_config)
        fullfile = os.path.join(driver_config['harvester']['directory'],
                            driver_config['harvester']['pattern'])
        if os.path.exists(fullfile):
            os.remove(fullfile)

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        self.clean_file()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml',
                         count=1, timeout=10)

        # there is only one file we read from, this example 'appends' data to
        # the end of the node59p1.dat file, and the data from the new append
        # is returned (not including the original data from _step1)
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1, timeout=10)

        # now 'appends' the rest of the data and just check if we get the right number
        self.clear_async_data()
        self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, count=4, timeout=10)

        self.driver.stop_sampling()
        # reset the parser and harvester states
        self.driver.clear_states()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, count=1, timeout=10)

    def test_harvester_new_file_exception(self):
        """
        Test an exception raised after the driver is started during
        the file read.  Should call the exception callback.
        """
        self.clean_file()

        # create the file so that it is unreadable
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.assert_exception(IOError)

        # At this point the harvester thread is dead.  The agent
        # exception handle should handle this case.

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        self.clean_file()

        # Create and store the new driver state
        self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 314,
                                                        'last_checksum': '515e5da08a6b4bb0d197e62c410da532'},
                        DataSourceConfigKey.PARSER: {'in_process_data': [],
                                                     'unprocessed_data':[[0,69]],
                                                     'timestamp': 3583581301.0}}
        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1, timeout=10)

    def test_sequences(self):
        """
        Test new sequence flags are set correctly
        """
        self.clean_file()

        self.driver.start_sampling()

        self.clear_async_data()

        # step 2 contains 2 blocks, start with this and get both since we used them
        # separately in other tests (no new sequences)
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml',
                         count=2, timeout=10)

        # This file has had a section of FL data replaced with 0s, this should start a new
        # sequence for the data following the missing AD data
        self.clear_async_data()
        self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
                         count=3, timeout=10)

        # Now fill in the zeroed section from step3, this should just return the new
        # data with a new sequence flag
        self.clear_async_data()
        self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
                         count=1, timeout=10)

        # start over now, using step 4, make sure sequence flags just account for
        # missing data in file (there are some sections of bad data that don't
        # match in headers
        self.driver.stop_sampling()
        # reset the parser and harvester states
        self.driver.clear_states()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml',
                         count=6, timeout=10)
예제 #2
0
class IntegrationTest(DataSetIntegrationTestCase):

    def clean_file(self):
        # remove just the file we are using
        driver_config = self._driver_config()['startup_config']
        log.debug('startup config %s', driver_config)
        fullfile = os.path.join(driver_config['harvester']['directory'],
                            driver_config['harvester']['pattern'])
        if os.path.exists(fullfile):
            os.remove(fullfile)

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        self.clean_file()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml',
                         count=1, timeout=10)

        # there is only one file we read from, this example 'appends' data to
        # the end of the node59p1.dat file, and the data from the new append
        # is returned (not including the original data from _step1)
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1, timeout=10)

        # now 'appends' the rest of the data and just check if we get the right number
        self.clear_async_data()
        self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, count=4, timeout=10)

        self.driver.stop_sampling()
        
        # Reset the driver with no memento
        self.memento = None
        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        self.driver.start_sampling()

        self.clear_async_data()
        
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, count=1, timeout=10)

    def test_harvester_new_file_exception(self):
        """
        Test an exception raised after the driver is started during
        the file read.  Should call the exception callback.
        """
        self.clean_file()

        # create the file so that it is unreadable
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.assert_exception(ValueError)

        # At this point the harvester thread is dead.  The agent
        # exception handle should handle this case.

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        self.clean_file()
        self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
        driver_config = self._driver_config()['startup_config']
        fullfile = os.path.join(driver_config['harvester']['directory'],
                            driver_config['harvester']['pattern'])
        mod_time = os.path.getmtime(fullfile)

        # Create and store the new driver state
        self.memento = {DriverStateKey.FILE_SIZE: 314,
                        DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
                        DriverStateKey.FILE_MOD_DATE: mod_time,
                        DriverStateKey.PARSER_STATE: {'in_process_data': [],
                                                     'unprocessed_data':[[0,69]],
                                                     'timestamp': 3583581301.0}
                        }

        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1, timeout=10)

    def test_sequences(self):
        """
        Test new sequence flags are set correctly
        """
        self.clean_file()

        self.driver.start_sampling()

        self.clear_async_data()

        # step 2 contains 2 blocks, start with this and get both since we used them
        # separately in other tests (no new sequences)
        self.clear_async_data()
        self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml',
                         count=2, timeout=10)

        # This file has had a section of FL data replaced with 0s, this should start a new
        # sequence for the data following the missing AD data
        self.clear_async_data()
        self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
                         count=3, timeout=10)

        # Now fill in the zeroed section from step3, this should just return the new
        # data with a new sequence flag
        self.clear_async_data()
        self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
                         count=1, timeout=10)

        # start over now, using step 4, make sure sequence flags just account for
        # missing data in file (there are some sections of bad data that don't
        # match in headers
        self.driver.stop_sampling()

        # Reset the driver with no memento
        self.memento = None
        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml',
                         count=6, timeout=10)
예제 #3
0
class IntegrationTest(DataSetIntegrationTestCase):

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        # Start sampling
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1.txt.result.yml', count=2, timeout=10)

        # there is only one file we read from, this example 'appends' data to
        # the end of the node59p1.dat file, and the data from the new append
        # is returned (not including the original data from _step1)
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1)

        # now 'appends' the rest of the data and just check if we get the right number
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step4.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle, count=4)

    def test_harvester_new_file_exception(self):
        """
        Test an exception raised after the driver is started during
        the file read.  Should call the exception callback.
        """
        # create the file so that it is unreadable
        self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
                                        mode=000, copy_metadata=False)

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.assert_exception(ValueError)

        # At this point the harvester thread is dead.  The agent
        # exception handle should handle this case.

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        driver_config = self._driver_config()['startup_config']
        dosta_telem_config = driver_config['harvester'][DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
        fullfile = os.path.join(dosta_telem_config['directory'], dosta_telem_config['pattern'])
        mod_time = os.path.getmtime(fullfile)

        # Create and store the new driver state
        self.memento = {
            DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
                "node59p1.dat": {
                    DriverStateKey.FILE_SIZE: 314,
                    DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
                    DriverStateKey.FILE_MOD_DATE: mod_time,
                    DriverStateKey.PARSER_STATE: {
                        'in_process_data': [],
                        'unprocessed_data':[[0,69]]
                    }
                }
            }
        }

        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
                         count=1)

    def test_back_fill(self):
        """
        Test new sequence flags are set correctly
        """
        self.driver.start_sampling()

        # step 2 contains 2 blocks, start with this and get both since we used them
        # separately in other tests 
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml', count=3)

        # This file has had a section of DO data replaced with 0s
        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
                         count=3)

        # Now fill in the zeroed section from step3, this should just return the new
        # data
        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
                         count=1)

        # start over now, using step 4
        self.driver.stop_sampling()

        # Reset the driver with no memento
        self.memento = None
        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'],
            self.memento,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-4.txt.result.yml', count=7)

    def test_all_good(self):
        """
        Test that a set of data with no bad data, where there is no remaining
        unprocessed data in between
        """
        self.driver.start_sampling()

        self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat")
        self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml', count=3)

        self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml',
                         count=1)

    def test_all_good(self):
        """
        Test that a set of data with no bad data, where there is no remaining
        unprocessed data in between
        """
        self.driver.start_sampling()

        self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat")
        self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml', count=3)

        self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat")
        self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml',
                         count=1)
예제 #4
0
class IntegrationTest(DataSetIntegrationTestCase):
    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver
        sampling can be started and stopped
        """
        # Start sampling
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step1.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(
            (DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1.txt.result.yml',
            count=2,
            timeout=10)

        # there is only one file we read from, this example 'appends' data to
        # the end of the node59p1.dat file, and the data from the new append
        # is returned (not including the original data from _step1)
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle,
                         'test_data_2.txt.result.yml',
                         count=1)

        # now 'appends' the rest of the data and just check if we get the right number
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step4.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle, count=4)

    def test_harvester_new_file_exception(self):
        """
        Test an exception raised after the driver is started during
        the file read.  Should call the exception callback.
        """
        # create the file so that it is unreadable
        self.create_sample_data_set_dir("node59p1_step1.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        mode=000,
                                        copy_metadata=False)

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.assert_exception(ValueError)

        # At this point the harvester thread is dead.  The agent
        # exception handle should handle this case.

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        self.create_sample_data_set_dir("node59p1_step1.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        driver_config = self._driver_config()['startup_config']
        dosta_telem_config = driver_config['harvester'][
            DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
        fullfile = os.path.join(dosta_telem_config['directory'],
                                dosta_telem_config['pattern'])
        mod_time = os.path.getmtime(fullfile)

        # Create and store the new driver state
        self.memento = {
            DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
                "node59p1.dat": {
                    DriverStateKey.FILE_SIZE: 314,
                    DriverStateKey.FILE_CHECKSUM:
                    '515e5da08a6b4bb0d197e62c410da532',
                    DriverStateKey.FILE_MOD_DATE: mod_time,
                    DriverStateKey.PARSER_STATE: {
                        StateKey.IN_PROCESS_DATA: [],
                        StateKey.UNPROCESSED_DATA: [[0, 69]],
                        StateKey.FILE_SIZE: 314
                    }
                }
            }
        }

        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(DostadParserDataParticle,
                         'test_data_2.txt.result.yml',
                         count=1)

    def test_back_fill(self):
        """
        Test new sequence flags are set correctly
        """
        self.driver.start_sampling()

        # step 2 contains 2 blocks, start with this and get both since we used them
        # separately in other tests
        self.clear_async_data()
        self.create_sample_data_set_dir("node59p1_step2.dat",
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(
            (DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml',
            count=3)

        # This file has had a section of DO data replaced with 0s
        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step3.dat',
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle,
                         'test_data_3.txt.result.yml',
                         count=3)

        # Now fill in the zeroed section from step3, this should just return the new
        # data
        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step4.dat',
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(DostadParserDataParticle,
                         'test_data_4.txt.result.yml',
                         count=1)

        # start over now, using step 4
        self.driver.stop_sampling()

        # Reset the driver with no memento
        self.memento = None
        self.driver = MflmDOSTADDataSetDriver(
            self._driver_config()['startup_config'], self.memento,
            self.data_callback, self.state_callback, self.event_callback,
            self.exception_callback)
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data_set_dir('node59p1_step4.dat',
                                        TELEM_DIR,
                                        "node59p1.dat",
                                        copy_metadata=False)
        self.assert_data(
            (DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-4.txt.result.yml',
            count=7)

    def test_all_good(self):
        """
        Test that a set of data with no bad data, where there is no remaining
        unprocessed data in between
        """
        self.driver.start_sampling()

        self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR,
                                        "node59p1.dat")
        self.assert_data(
            (DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml',
            count=3)

        self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR,
                                        "node59p1.dat")
        self.assert_data(DostadParserDataParticle,
                         'test_data_all_good.txt.result.yml',
                         count=1)

    def test_all_good(self):
        """
        Test that a set of data with no bad data, where there is no remaining
        unprocessed data in between
        """
        self.driver.start_sampling()

        self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR,
                                        "node59p1.dat")
        self.assert_data(
            (DostadParserDataParticle, DostadMetadataDataParticle),
            'test_data_1-2.txt.result.yml',
            count=3)

        self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR,
                                        "node59p1.dat")
        self.assert_data(DostadParserDataParticle,
                         'test_data_all_good.txt.result.yml',
                         count=1)