def test_stop_resume(self): """ Test the ability to stop and restart the process """ # create sample data files self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") # get file metadata for use in the state dictionary startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY) file_path_1 = os.path.join(directory, "DATA001.TXT") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(directory, "DATA002.TXT") mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state state = { "DATA001.TXT": { 'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.TXT": { 'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': { 'position': 201, 'timestamp': 3575062804.0 } } } self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], state, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = WfpCTDPFKDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback)
def test_stop_resume(self): """ Test the ability to stop and restart the process """ # create sample data files self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") # get file metadata for use in the state dictionary startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY) file_path_1 = os.path.join(directory, "DATA001.TXT") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(directory, "DATA002.TXT") mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state state = {"DATA001.TXT":{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.TXT":{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 201, 'timestamp': 3575062804.0} } } self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], state, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
def test_stop_resume(self): """ Test the ability to stop and restart the process """ # Create and store the new driver state #""" self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/DATA001.TXT', DataSourceConfigKey.PARSER: {'position': 201, 'timestamp': 3575062804.0}} self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) #""" # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase): def setUp(self): super(IntegrationTest, self).setUp() def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = WfpCTDPFKDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped. """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10) self.clear_async_data() self.create_sample_data('C0000181.TXT', "DATA003.TXT") self.assert_data(CtdpfkParserDataParticle, count=303, timeout=300) # 20 self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA004.TXT") self.assert_data(CtdpfkParserDataParticle, count=1, timeout=10) def test_stop_resume(self): """ Test the ability to stop and restart the process """ # create sample data files self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") # get file metadata for use in the state dictionary startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get( DataSetDriverConfigKeys.DIRECTORY) file_path_1 = os.path.join(directory, "DATA001.TXT") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(directory, "DATA002.TXT") mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state state = { "DATA001.TXT": { 'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.TXT": { 'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': { 'position': 201, 'timestamp': 3575062804.0 } } } self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], state, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.driver.wfp.ctdpfk.driver import WfpCTDPFKDataSetDriver from mi.dataset.parser.test.test_ctdpfk import CtdpfkParserUnitTestCase from pyon.agent.agent import ResourceAgentState from interface.objects import CapabilityType from interface.objects import AgentCapability from interface.objects import ResourceAgentErrorEvent from interface.objects import ResourceAgentConnectionLostErrorEvent DataSetTestCase.initialize( driver_module='mi.dataset.driver.wfp.ctdpfk.driver', driver_class="WfpCTDPFKDataSetDriver", agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=WfpCTDPFKDataSetDriver.stream_config(), startup_config={ DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.TXT', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} }) SAMPLE_STREAM = 'ctdpfk_parsed' ############################################################################### # INT TESTS # # Device specific integration tests are for #
class IntegrationTest(DataSetIntegrationTestCase): def setUp(self): super(IntegrationTest, self).setUp() def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = WfpCTDPFKDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped. """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10) self.clear_async_data() self.create_sample_data('C0000181.TXT', "DATA003.TXT") self.assert_data(CtdpfkParserDataParticle, count=303, timeout=300) # 20 self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA004.TXT") self.assert_data(CtdpfkParserDataParticle, count=1, timeout=10) def test_stop_resume(self): """ Test the ability to stop and restart the process """ # create sample data files self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") # get file metadata for use in the state dictionary startup_config = self._driver_config()['startup_config'] directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY) file_path_1 = os.path.join(directory, "DATA001.TXT") # need to reset file mod time since file is created again mod_time_1 = os.path.getmtime(file_path_1) file_size_1 = os.path.getsize(file_path_1) with open(file_path_1) as filehandle: md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest() file_path_2 = os.path.join(directory, "DATA002.TXT") mod_time_2 = os.path.getmtime(file_path_2) file_size_2 = os.path.getsize(file_path_2) with open(file_path_2) as filehandle: md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest() # Create and store the new driver state state = {"DATA001.TXT":{'ingested': True, 'file_mod_date': mod_time_1, 'file_checksum': md5_checksum_1, 'file_size': file_size_1, 'parser_state': {} }, "DATA002.TXT":{'ingested': False, 'file_mod_date': mod_time_2, 'file_checksum': md5_checksum_2, 'file_size': file_size_2, 'parser_state': {'position': 201, 'timestamp': 3575062804.0} } } self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], state, self.data_callback, self.state_callback, self.exception_callback) # create some data to parse self.clear_async_data() self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.parser.test.test_ctdpfk import CtdpfkParserUnitTestCase from pyon.agent.agent import ResourceAgentState from interface.objects import CapabilityType from interface.objects import AgentCapability from interface.objects import ResourceAgentErrorEvent from interface.objects import ResourceAgentConnectionLostErrorEvent DataSetTestCase.initialize( driver_module='mi.dataset.driver.wfp.ctdpfk.driver', driver_class="WfpCTDPFKDataSetDriver", agent_resource_id = '123xyz', agent_name = 'Agent007', agent_packet_config = WfpCTDPFKDataSetDriver.stream_config(), startup_config = { DataSourceConfigKey.HARVESTER: { DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest', DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest', DataSetDriverConfigKeys.PATTERN: '*.TXT', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataSourceConfigKey.PARSER: {} } ) SAMPLE_STREAM='ctdpfk_parsed' ###############################################################################
class IntegrationTest(DataSetIntegrationTestCase): def setUp(self): super(IntegrationTest, self).setUp() def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = WfpCTDPFKDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_parameters(self): """ Verify that we can get, set, and report all driver parameters. """ expected_params = [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.PUBLISHER_POLLING_INTERVAL, DriverParameter.RECORDS_PER_SECOND] (res_cmds, res_params) = self.driver.get_resource_capabilities() # Ensure capabilities are as expected self.assertEqual(len(res_cmds), 0) self.assertEqual(len(res_params), len(expected_params)) self.assertEqual(sorted(res_params), sorted(expected_params)) # Verify default values are as expected. params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Try set resource individually self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 2}) self.driver.set_resource({DriverParameter.PUBLISHER_POLLING_INTERVAL: 2}) self.driver.set_resource({DriverParameter.RECORDS_PER_SECOND: 59}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 2) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 59) # Try set resource in bulk self.driver.set_resource( {DriverParameter.BATCHED_PARTICLE_COUNT: 1, DriverParameter.PUBLISHER_POLLING_INTERVAL: .1, DriverParameter.RECORDS_PER_SECOND: 60}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Set with some bad values with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 'a'}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: -1}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 0}) # Try to configure with the driver startup config driver_config = self._driver_config()['startup_config'] cfg = { DataSourceConfigKey.HARVESTER: driver_config.get(DataSourceConfigKey.HARVESTER), DataSourceConfigKey.PARSER: driver_config.get(DataSourceConfigKey.PARSER), DataSourceConfigKey.DRIVER: { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, } } self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 3) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 3) # Finally verify we get a KeyError when sending in bad config keys cfg[DataSourceConfigKey.DRIVER] = { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, 'something_extra': 1 } with self.assertRaises(KeyError): self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped. """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10) self.clear_async_data() self.create_sample_data('C0000181.TXT', "DATA003.TXT") self.assert_data(CtdpfkParserDataParticle, count=303, timeout=300) # 20 self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA004.TXT") self.assert_data(CtdpfkParserDataParticle, count=1, timeout=10) def test_stop_resume(self): """ Test the ability to stop and restart the process """ # Create and store the new driver state #""" self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/DATA001.TXT', DataSourceConfigKey.PARSER: {'position': 201, 'timestamp': 3575062804.0}} self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) #""" # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
def test_parameters(self): """ Verify that we can get, set, and report all driver parameters. """ expected_params = [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.PUBLISHER_POLLING_INTERVAL, DriverParameter.RECORDS_PER_SECOND] (res_cmds, res_params) = self.driver.get_resource_capabilities() # Ensure capabilities are as expected self.assertEqual(len(res_cmds), 0) self.assertEqual(len(res_params), len(expected_params)) self.assertEqual(sorted(res_params), sorted(expected_params)) # Verify default values are as expected. params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Try set resource individually self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 2}) self.driver.set_resource({DriverParameter.PUBLISHER_POLLING_INTERVAL: 2}) self.driver.set_resource({DriverParameter.RECORDS_PER_SECOND: 59}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 2) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 59) # Try set resource in bulk self.driver.set_resource( {DriverParameter.BATCHED_PARTICLE_COUNT: 1, DriverParameter.PUBLISHER_POLLING_INTERVAL: .1, DriverParameter.RECORDS_PER_SECOND: 60}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Set with some bad values with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 'a'}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: -1}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 0}) # Try to configure with the driver startup config driver_config = self._driver_config()['startup_config'] cfg = { DataSourceConfigKey.HARVESTER: driver_config.get(DataSourceConfigKey.HARVESTER), DataSourceConfigKey.PARSER: driver_config.get(DataSourceConfigKey.PARSER), DataSourceConfigKey.DRIVER: { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, } } self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 3) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 3) # Finally verify we get a KeyError when sending in bad config keys cfg[DataSourceConfigKey.DRIVER] = { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, 'something_extra': 1 } with self.assertRaises(KeyError): self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback)