def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(
DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {
"DATA001.TXT": {
'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT": {
'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {
'position': 201,
'timestamp': 3575062804.0
}
}
}
# Create and store the new driver state
self.driver = WfpEngineeringDataSetDriver(
self._driver_config()['startup_config'], state, self.data_callback,
self.state_callback, self.event_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpEngineeringDataParticle,
'test_data_3B.txt.partial_results.yml',
count=5,
timeout=10)
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpEngineeringDataSetDriver({}, self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpEngineeringDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {"DATA001.TXT":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 201, 'timestamp': 3575062804.0}
}
}
# Create and store the new driver state
self.driver = WfpEngineeringDataSetDriver(
self._driver_config()['startup_config'],
state,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpEngineeringDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpEngineeringDataSetDriver({}, self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
log.error("*** WfpEngineeringDataParticle = " +
repr(WfpEngineeringDataParticle))
self.assert_data(WfpEngineeringDataParticle,
'test_data_1.txt.result.yml',
count=1,
timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.assert_data(WfpEngineeringDataParticle,
'test_data_3.txt.result.yml',
count=8,
timeout=10)
self.clear_async_data()
self.create_sample_data('E0000152.TXT', "DATA003.TXT")
self.assert_data(WfpEngineeringDataParticle, count=11,
timeout=30) # 20
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA004.TXT")
self.assert_data(WfpEngineeringDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(
DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {
"DATA001.TXT": {
'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT": {
'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {
'position': 201,
'timestamp': 3575062804.0
}
}
}
# Create and store the new driver state
self.driver = WfpEngineeringDataSetDriver(
self._driver_config()['startup_config'], state, self.data_callback,
self.state_callback, self.event_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpEngineeringDataParticle,
'test_data_3B.txt.partial_results.yml',
count=5,
timeout=10)
from mi.dataset.parser.test.test_wfp_parser import WfpParserUnitTestCase
from mi.dataset.driver.wfp.engineering.driver import WfpEngineeringDataSetDriver
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.wfp.engineering.driver',
driver_class="WfpEngineeringDataSetDriver",
agent_resource_id='123xyz',
agent_name='Agent007',
agent_packet_config=WfpEngineeringDataSetDriver.stream_config(),
startup_config={
DataSourceConfigKey.HARVESTER: {
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: '*.TXT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
})
SAMPLE_STREAM = 'wfp_engineering_parsed'
###############################################################################
# INT TESTS #
# Device specific integration tests are for #
class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpEngineeringDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
log.error("*** WfpEngineeringDataParticle = " + repr(WfpEngineeringDataParticle))
self.assert_data(WfpEngineeringDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.assert_data(WfpEngineeringDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10)
self.clear_async_data()
self.create_sample_data('E0000152.TXT', "DATA003.TXT")
self.assert_data(WfpEngineeringDataParticle, count=11, timeout=30) # 20
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA004.TXT")
self.assert_data(WfpEngineeringDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {"DATA001.TXT":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 201, 'timestamp': 3575062804.0}
}
}
# Create and store the new driver state
self.driver = WfpEngineeringDataSetDriver(
self._driver_config()['startup_config'],
state,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpEngineeringDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.driver.wfp.engineering.driver import WfpEngineeringDataSetDriver
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.wfp.engineering.driver',
driver_class="WfpEngineeringDataSetDriver",
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = WfpEngineeringDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: '*.TXT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
}
)
SAMPLE_STREAM = 'wfp_engineering_parsed'
###############################################################################