def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = IssmRiNUTNRBDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data('test_data_1.log', "DATA001.log")
self.create_sample_data('test_data_3.log', "DATA002.log")
# get file metadata for use in state memento
startup_config = self._driver_config()['startup_config']
file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY),
"DATA001.log")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY),
"DATA002.log")
# need to reset file mod time since file is created again
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state, after completed reading "DATA001.log"
# Note, since file "DATA001.log" is ingested, parser state is not looked at, in a real run there would be a state in there
self.memento = {"DATA001.log":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.log":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 2628}
}
}
self.driver = IssmRiNUTNRBDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(NutnrbDataParticle, 'test_data_3.log.partial_results.yml', count=5, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = IssmRiNUTNRBDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.log', "DATA001.log")
self.assert_data(NutnrbDataParticle, 'test_data_1.log.result.yml', count=1, timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.log', "DATA002.log")
self.assert_data(NutnrbDataParticle, 'test_data_3.log.result.yml', count=8, timeout=10)
self.clear_async_data()
self.create_sample_data('20121213.nutnr.log', "DATA003.log")
self.assert_data(NutnrbDataParticle, count=435, timeout=20)
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.log', "DATA004.log")
self.assert_data(NutnrbDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data('test_data_1.log', "DATA001.log")
self.create_sample_data('test_data_3.log', "DATA002.log")
# get file metadata for use in state memento
startup_config = self._driver_config()['startup_config']
file_path_1 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY),
"DATA001.log")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY),
"DATA002.log")
# need to reset file mod time since file is created again
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state, after completed reading "DATA001.log"
# Note, since file "DATA001.log" is ingested, parser state is not looked at, in a real run there would be a state in there
self.memento = {"DATA001.log":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.log":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 2628}
}
}
self.driver = IssmRiNUTNRBDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(NutnrbDataParticle, 'test_data_3.log.partial_results.yml', count=5, timeout=10)
from mi.core.instrument.instrument_driver import DriverEvent
from mi.dataset.parser.nutnrb import NutnrbDataParticle
from mi.dataset.driver.issm.nutnrb.driver import IssmRiNUTNRBDataSetDriver
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.issm.nutnrb.driver',
driver_class='IssmRiNUTNRBDataSetDriver',
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = IssmRiNUTNRBDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: '*.log',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
}
)
SAMPLE_STREAM = 'nutnrb_parsed'
###############################################################################
