class IntegrationTest(DataSetIntegrationTestCase): def setUp(self): super(IntegrationTest, self).setUp() def test_harvester_config_exception(self): """ Start the a driver with a bad configuration. Should raise an exception. """ with self.assertRaises(ConfigurationException): self.driver = WfpCTDPFKDataSetDriver({}, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_parameters(self): """ Verify that we can get, set, and report all driver parameters. """ expected_params = [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.PUBLISHER_POLLING_INTERVAL, DriverParameter.RECORDS_PER_SECOND] (res_cmds, res_params) = self.driver.get_resource_capabilities() # Ensure capabilities are as expected self.assertEqual(len(res_cmds), 0) self.assertEqual(len(res_params), len(expected_params)) self.assertEqual(sorted(res_params), sorted(expected_params)) # Verify default values are as expected. params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Try set resource individually self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 2}) self.driver.set_resource({DriverParameter.PUBLISHER_POLLING_INTERVAL: 2}) self.driver.set_resource({DriverParameter.RECORDS_PER_SECOND: 59}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 2) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 59) # Try set resource in bulk self.driver.set_resource( {DriverParameter.BATCHED_PARTICLE_COUNT: 1, DriverParameter.PUBLISHER_POLLING_INTERVAL: .1, DriverParameter.RECORDS_PER_SECOND: 60}) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .1) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60) # Set with some bad values with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 'a'}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: -1}) with self.assertRaises(InstrumentParameterException): self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 0}) # Try to configure with the driver startup config driver_config = self._driver_config()['startup_config'] cfg = { DataSourceConfigKey.HARVESTER: driver_config.get(DataSourceConfigKey.HARVESTER), DataSourceConfigKey.PARSER: driver_config.get(DataSourceConfigKey.PARSER), DataSourceConfigKey.DRIVER: { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, } } self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback) params = self.driver.get_resource(DriverParameter.ALL) log.debug("Get Resources Result: %s", params) self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 3) self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .2) self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 3) # Finally verify we get a KeyError when sending in bad config keys cfg[DataSourceConfigKey.DRIVER] = { DriverParameter.PUBLISHER_POLLING_INTERVAL: .2, DriverParameter.RECORDS_PER_SECOND: 3, DriverParameter.BATCHED_PARTICLE_COUNT: 3, 'something_extra': 1 } with self.assertRaises(KeyError): self.driver = WfpCTDPFKDataSetDriver( cfg, self.memento, self.data_callback, self.state_callback, self.exception_callback) def test_get(self): """ Test that we can get data from files. Verify that the driver sampling can be started and stopped. """ self.clear_sample_data() # Start sampling and watch for an exception self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10) self.clear_async_data() self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10) self.clear_async_data() self.create_sample_data('C0000181.TXT', "DATA003.TXT") self.assert_data(CtdpfkParserDataParticle, count=303, timeout=300) # 20 self.driver.stop_sampling() self.driver.start_sampling() self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA004.TXT") self.assert_data(CtdpfkParserDataParticle, count=1, timeout=10) def test_stop_resume(self): """ Test the ability to stop and restart the process """ # Create and store the new driver state #""" self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/DATA001.TXT', DataSourceConfigKey.PARSER: {'position': 201, 'timestamp': 3575062804.0}} self.driver = WfpCTDPFKDataSetDriver( self._driver_config()['startup_config'], self.memento, self.data_callback, self.state_callback, self.exception_callback) #""" # create some data to parse self.clear_async_data() self.create_sample_data('test_data_1.txt', "DATA001.TXT") self.create_sample_data('test_data_3.txt', "DATA002.TXT") self.driver.start_sampling() # verify data is produced self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)