class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpCTDPFKDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_parameters(self):
"""
Verify that we can get, set, and report all driver parameters.
"""
expected_params = [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.PUBLISHER_POLLING_INTERVAL, DriverParameter.RECORDS_PER_SECOND]
(res_cmds, res_params) = self.driver.get_resource_capabilities()
# Ensure capabilities are as expected
self.assertEqual(len(res_cmds), 0)
self.assertEqual(len(res_params), len(expected_params))
self.assertEqual(sorted(res_params), sorted(expected_params))
# Verify default values are as expected.
params = self.driver.get_resource(DriverParameter.ALL)
log.debug("Get Resources Result: %s", params)
self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1)
self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 1)
self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60)
# Try set resource individually
self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 2})
self.driver.set_resource({DriverParameter.PUBLISHER_POLLING_INTERVAL: 2})
self.driver.set_resource({DriverParameter.RECORDS_PER_SECOND: 59})
params = self.driver.get_resource(DriverParameter.ALL)
log.debug("Get Resources Result: %s", params)
self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 2)
self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], 2)
self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 59)
# Try set resource in bulk
self.driver.set_resource(
{DriverParameter.BATCHED_PARTICLE_COUNT: 1,
DriverParameter.PUBLISHER_POLLING_INTERVAL: .1,
DriverParameter.RECORDS_PER_SECOND: 60})
params = self.driver.get_resource(DriverParameter.ALL)
log.debug("Get Resources Result: %s", params)
self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 1)
self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .1)
self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 60)
# Set with some bad values
with self.assertRaises(InstrumentParameterException):
self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 'a'})
with self.assertRaises(InstrumentParameterException):
self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: -1})
with self.assertRaises(InstrumentParameterException):
self.driver.set_resource({DriverParameter.BATCHED_PARTICLE_COUNT: 0})
# Try to configure with the driver startup config
driver_config = self._driver_config()['startup_config']
cfg = {
DataSourceConfigKey.HARVESTER: driver_config.get(DataSourceConfigKey.HARVESTER),
DataSourceConfigKey.PARSER: driver_config.get(DataSourceConfigKey.PARSER),
DataSourceConfigKey.DRIVER: {
DriverParameter.PUBLISHER_POLLING_INTERVAL: .2,
DriverParameter.RECORDS_PER_SECOND: 3,
DriverParameter.BATCHED_PARTICLE_COUNT: 3,
}
}
self.driver = WfpCTDPFKDataSetDriver(
cfg,
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
params = self.driver.get_resource(DriverParameter.ALL)
log.debug("Get Resources Result: %s", params)
self.assertEqual(params[DriverParameter.BATCHED_PARTICLE_COUNT], 3)
self.assertEqual(params[DriverParameter.PUBLISHER_POLLING_INTERVAL], .2)
self.assertEqual(params[DriverParameter.RECORDS_PER_SECOND], 3)
# Finally verify we get a KeyError when sending in bad config keys
cfg[DataSourceConfigKey.DRIVER] = {
DriverParameter.PUBLISHER_POLLING_INTERVAL: .2,
DriverParameter.RECORDS_PER_SECOND: 3,
DriverParameter.BATCHED_PARTICLE_COUNT: 3,
'something_extra': 1
}
with self.assertRaises(KeyError):
self.driver = WfpCTDPFKDataSetDriver(
cfg,
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.assert_data(CtdpfkParserDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10)
self.clear_async_data()
self.create_sample_data('C0000181.TXT', "DATA003.TXT")
self.assert_data(CtdpfkParserDataParticle, count=303, timeout=300) # 20
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA004.TXT")
self.assert_data(CtdpfkParserDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# Create and store the new driver state
#"""
self.memento = {DataSourceConfigKey.HARVESTER: '/tmp/dsatest/DATA001.TXT',
DataSourceConfigKey.PARSER: {'position': 201, 'timestamp': 3575062804.0}}
self.driver = WfpCTDPFKDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
#"""
# create some data to parse
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdpfkParserDataParticle, 'test_data_3.txt.partial_results.yml', count=5, timeout=10)
