def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpFLORTKDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {"DATA001.TXT":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 201, 'timestamp': 3575062804.0}
}
}
self.driver = WfpFLORTKDataSetDriver(
self._driver_config()['startup_config'],
state,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpFlortkDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpFLORTKDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpFLORTKDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
log.error("*** WfpFlortkDataParticle = " + repr(WfpFlortkDataParticle))
self.assert_data(WfpFlortkDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.assert_data(WfpFlortkDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10)
self.clear_async_data()
self.create_sample_data('E0000152.TXT', "DATA003.TXT")
self.assert_data(WfpFlortkDataParticle, count=11, timeout=30) # 20
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA004.TXT")
self.assert_data(WfpFlortkDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {"DATA001.TXT":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 201, 'timestamp': 3575062804.0}
}
}
self.driver = WfpFLORTKDataSetDriver(
self._driver_config()['startup_config'],
state,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpFlortkDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.driver.wfp.flortk.driver import WfpFLORTKDataSetDriver
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.wfp.flortk.driver',
driver_class="WfpFLORTKDataSetDriver",
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = WfpFLORTKDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: '*.TXT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
}
)
SAMPLE_STREAM='wfp_flort_k_parsed'
###############################################################################
class IntegrationTest(DataSetIntegrationTestCase):
def setUp(self):
super(IntegrationTest, self).setUp()
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpFLORTKDataSetDriver({},
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver sampling
can be started and stopped.
"""
self.clear_sample_data()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
log.error("*** WfpFlortkDataParticle = " + repr(WfpFlortkDataParticle))
self.assert_data(WfpFlortkDataParticle, 'test_data_1.txt.result.yml', count=1, timeout=10)
self.clear_async_data()
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
self.assert_data(WfpFlortkDataParticle, 'test_data_3.txt.result.yml', count=8, timeout=10)
self.clear_async_data()
self.create_sample_data('E0000152.TXT', "DATA003.TXT")
self.assert_data(WfpFlortkDataParticle, count=11, timeout=30) # 20
self.driver.stop_sampling()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('test_data_1.txt', "DATA004.TXT")
self.assert_data(WfpFlortkDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {"DATA001.TXT":{'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT":{'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {'position': 201, 'timestamp': 3575062804.0}
}
}
self.driver = WfpFLORTKDataSetDriver(
self._driver_config()['startup_config'],
state,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(WfpFlortkDataParticle, 'test_data_3B.txt.partial_results.yml', count=5, timeout=10)
from mi.dataset.driver.wfp.flortk.driver import WfpFLORTKDataSetDriver
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.wfp.flortk.driver',
driver_class="WfpFLORTKDataSetDriver",
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = WfpFLORTKDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: '*.TXT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
}
)
SAMPLE_STREAM='wfp_flort_k_parsed'
###############################################################################
