def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        # create sample data files
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        # get file metadata for use in the state dictionary
        startup_config = self._driver_config()['startup_config']
        directory = startup_config[DataSourceConfigKey.HARVESTER].get(
            DataSetDriverConfigKeys.DIRECTORY)
        file_path_1 = os.path.join(directory, "DATA001.TXT")
        # need to reset file mod time since file is created again
        mod_time_1 = os.path.getmtime(file_path_1)
        file_size_1 = os.path.getsize(file_path_1)
        with open(file_path_1) as filehandle:
            md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
        file_path_2 = os.path.join(directory, "DATA002.TXT")
        mod_time_2 = os.path.getmtime(file_path_2)
        file_size_2 = os.path.getsize(file_path_2)
        with open(file_path_2) as filehandle:
            md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()

        # Create and store the new driver state
        state = {
            "DATA001.TXT": {
                'ingested': True,
                'file_mod_date': mod_time_1,
                'file_checksum': md5_checksum_1,
                'file_size': file_size_1,
                'parser_state': {}
            },
            "DATA002.TXT": {
                'ingested': False,
                'file_mod_date': mod_time_2,
                'file_checksum': md5_checksum_2,
                'file_size': file_size_2,
                'parser_state': {
                    'position': 201,
                    'timestamp': 3575062804.0
                }
            }
        }
        # Create and store the new driver state
        self.driver = WfpEngineeringDataSetDriver(
            self._driver_config()['startup_config'], state, self.data_callback,
            self.state_callback, self.event_callback, self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(WfpEngineeringDataParticle,
                         'test_data_3B.txt.partial_results.yml',
                         count=5,
                         timeout=10)
 def test_harvester_config_exception(self):
     """
     Start the a driver with a bad configuration.  Should raise
     an exception.
     """
     with self.assertRaises(ConfigurationException):
         self.driver = WfpEngineeringDataSetDriver({}, self.memento,
                                                   self.data_callback,
                                                   self.state_callback,
                                                   self.event_callback,
                                                   self.exception_callback)
 def test_harvester_config_exception(self):
     """
     Start the a driver with a bad configuration.  Should raise
     an exception.
     """
     with self.assertRaises(ConfigurationException):
         self.driver = WfpEngineeringDataSetDriver({},
             self.memento,
             self.data_callback,
             self.state_callback,
             self.exception_callback)
    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        # create sample data files
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        # get file metadata for use in the state dictionary
        startup_config = self._driver_config()['startup_config']
        directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
        file_path_1 = os.path.join(directory, "DATA001.TXT")
        # need to reset file mod time since file is created again
        mod_time_1 = os.path.getmtime(file_path_1)
        file_size_1 = os.path.getsize(file_path_1)
        with open(file_path_1) as filehandle:
            md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
        file_path_2 = os.path.join(directory, "DATA002.TXT")
        mod_time_2 = os.path.getmtime(file_path_2)
        file_size_2 = os.path.getsize(file_path_2)
        with open(file_path_2) as filehandle:
            md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()

        # Create and store the new driver state
        state = {"DATA001.TXT":{'ingested': True,
                                'file_mod_date': mod_time_1,
                                'file_checksum': md5_checksum_1,
                                'file_size': file_size_1,
                                'parser_state': {}
                            },
                "DATA002.TXT":{'ingested': False,
                               'file_mod_date': mod_time_2,
                               'file_checksum': md5_checksum_2,
                               'file_size': file_size_2,
                               'parser_state': {'position': 201, 'timestamp': 3575062804.0}
                            }
        }
        # Create and store the new driver state
        self.driver = WfpEngineeringDataSetDriver(
            self._driver_config()['startup_config'],
            state,
            self.data_callback,
            self.state_callback,
            self.event_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(WfpEngineeringDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
    def setUp(self):
        super(IntegrationTest, self).setUp()

    def test_harvester_config_exception(self):
        """
        Start the a driver with a bad configuration.  Should raise
        an exception.
        """
        with self.assertRaises(ConfigurationException):
            self.driver = WfpEngineeringDataSetDriver({}, self.memento,
                                                      self.data_callback,
                                                      self.state_callback,
                                                      self.event_callback,
                                                      self.exception_callback)

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver sampling
        can be started and stopped.
        """
        self.clear_sample_data()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        log.error("*** WfpEngineeringDataParticle = " +
                  repr(WfpEngineeringDataParticle))
        self.assert_data(WfpEngineeringDataParticle,
                         'test_data_1.txt.result.yml',
                         count=1,
                         timeout=10)

        self.clear_async_data()
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        self.assert_data(WfpEngineeringDataParticle,
                         'test_data_3.txt.result.yml',
                         count=8,
                         timeout=10)

        self.clear_async_data()
        self.create_sample_data('E0000152.TXT', "DATA003.TXT")
        self.assert_data(WfpEngineeringDataParticle, count=11,
                         timeout=30)  # 20

        self.driver.stop_sampling()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.txt', "DATA004.TXT")
        self.assert_data(WfpEngineeringDataParticle, count=1, timeout=10)

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        # create sample data files
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        # get file metadata for use in the state dictionary
        startup_config = self._driver_config()['startup_config']
        directory = startup_config[DataSourceConfigKey.HARVESTER].get(
            DataSetDriverConfigKeys.DIRECTORY)
        file_path_1 = os.path.join(directory, "DATA001.TXT")
        # need to reset file mod time since file is created again
        mod_time_1 = os.path.getmtime(file_path_1)
        file_size_1 = os.path.getsize(file_path_1)
        with open(file_path_1) as filehandle:
            md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
        file_path_2 = os.path.join(directory, "DATA002.TXT")
        mod_time_2 = os.path.getmtime(file_path_2)
        file_size_2 = os.path.getsize(file_path_2)
        with open(file_path_2) as filehandle:
            md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()

        # Create and store the new driver state
        state = {
            "DATA001.TXT": {
                'ingested': True,
                'file_mod_date': mod_time_1,
                'file_checksum': md5_checksum_1,
                'file_size': file_size_1,
                'parser_state': {}
            },
            "DATA002.TXT": {
                'ingested': False,
                'file_mod_date': mod_time_2,
                'file_checksum': md5_checksum_2,
                'file_size': file_size_2,
                'parser_state': {
                    'position': 201,
                    'timestamp': 3575062804.0
                }
            }
        }
        # Create and store the new driver state
        self.driver = WfpEngineeringDataSetDriver(
            self._driver_config()['startup_config'], state, self.data_callback,
            self.state_callback, self.event_callback, self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(WfpEngineeringDataParticle,
                         'test_data_3B.txt.partial_results.yml',
                         count=5,
                         timeout=10)
from mi.dataset.parser.test.test_wfp_parser import WfpParserUnitTestCase
from mi.dataset.driver.wfp.engineering.driver import WfpEngineeringDataSetDriver

from pyon.agent.agent import ResourceAgentState

from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent

DataSetTestCase.initialize(
    driver_module='mi.dataset.driver.wfp.engineering.driver',
    driver_class="WfpEngineeringDataSetDriver",
    agent_resource_id='123xyz',
    agent_name='Agent007',
    agent_packet_config=WfpEngineeringDataSetDriver.stream_config(),
    startup_config={
        DataSourceConfigKey.HARVESTER: {
            DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
            DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
            DataSetDriverConfigKeys.PATTERN: '*.TXT',
            DataSetDriverConfigKeys.FREQUENCY: 1,
        },
        DataSourceConfigKey.PARSER: {}
    })

SAMPLE_STREAM = 'wfp_engineering_parsed'

###############################################################################
#                                INT TESTS                                    #
# Device specific integration tests are for                                   #
class IntegrationTest(DataSetIntegrationTestCase):
    def setUp(self):
        super(IntegrationTest, self).setUp()

    def test_harvester_config_exception(self):
        """
        Start the a driver with a bad configuration.  Should raise
        an exception.
        """
        with self.assertRaises(ConfigurationException):
            self.driver = WfpEngineeringDataSetDriver({},
                self.memento,
                self.data_callback,
                self.state_callback,
                self.exception_callback)

    def test_get(self):
        """
        Test that we can get data from files.  Verify that the driver sampling
        can be started and stopped.
        """
        self.clear_sample_data()

        # Start sampling and watch for an exception
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        log.error("*** WfpEngineeringDataParticle = " + repr(WfpEngineeringDataParticle))
        self.assert_data(WfpEngineeringDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)

        self.clear_async_data()
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        self.assert_data(WfpEngineeringDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10)

        self.clear_async_data()
        self.create_sample_data('E0000152.TXT', "DATA003.TXT")
        self.assert_data(WfpEngineeringDataParticle, count=11, timeout=30) # 20

        self.driver.stop_sampling()
        self.driver.start_sampling()

        self.clear_async_data()
        self.create_sample_data('test_data_1.txt', "DATA004.TXT")
        self.assert_data(WfpEngineeringDataParticle, count=1, timeout=10)

    def test_stop_resume(self):
        """
        Test the ability to stop and restart the process
        """
        # create sample data files
        self.create_sample_data('test_data_1.txt', "DATA001.TXT")
        self.create_sample_data('test_data_3.txt', "DATA002.TXT")
        # get file metadata for use in the state dictionary
        startup_config = self._driver_config()['startup_config']
        directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
        file_path_1 = os.path.join(directory, "DATA001.TXT")
        # need to reset file mod time since file is created again
        mod_time_1 = os.path.getmtime(file_path_1)
        file_size_1 = os.path.getsize(file_path_1)
        with open(file_path_1) as filehandle:
	    md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
        file_path_2 = os.path.join(directory, "DATA002.TXT")
        mod_time_2 = os.path.getmtime(file_path_2)
        file_size_2 = os.path.getsize(file_path_2)
        with open(file_path_2) as filehandle:
	    md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()

        # Create and store the new driver state
        state = {"DATA001.TXT":{'ingested': True,
                                'file_mod_date': mod_time_1,
                                'file_checksum': md5_checksum_1,
                                'file_size': file_size_1,
                                'parser_state': {}
                            },
                "DATA002.TXT":{'ingested': False,
                               'file_mod_date': mod_time_2,
                               'file_checksum': md5_checksum_2,
                               'file_size': file_size_2,
                               'parser_state': {'position': 201, 'timestamp': 3575062804.0}
                            }
        }
        # Create and store the new driver state
        self.driver = WfpEngineeringDataSetDriver(
            self._driver_config()['startup_config'],
            state,
            self.data_callback,
            self.state_callback,
            self.exception_callback)

        # create some data to parse
        self.clear_async_data()

        self.driver.start_sampling()

        # verify data is produced
        self.assert_data(WfpEngineeringDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.driver.wfp.engineering.driver import WfpEngineeringDataSetDriver

from pyon.agent.agent import ResourceAgentState

from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent

DataSetTestCase.initialize(
    driver_module='mi.dataset.driver.wfp.engineering.driver',
    driver_class="WfpEngineeringDataSetDriver",

    agent_resource_id = '123xyz',
    agent_name = 'Agent007',
    agent_packet_config = WfpEngineeringDataSetDriver.stream_config(),
    startup_config = {
        DataSourceConfigKey.HARVESTER:
        {
            DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
            DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
            DataSetDriverConfigKeys.PATTERN: '*.TXT',
            DataSetDriverConfigKeys.FREQUENCY: 1,
        },
        DataSourceConfigKey.PARSER: {}
    }
)

SAMPLE_STREAM = 'wfp_engineering_parsed'

###############################################################################