def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {
'last_filesize': 1300,
'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'
},
DataSourceConfigKey.PARSER: {
'in_process_data': [],
'unprocessed_data': [[0, 32], [222, 871], [1257, 1300]],
'timestamp': 3583725976.97
}
}
self.driver = MflmADCPSDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(AdcpsParserDataParticle,
'test_data_2.txt.result.yml',
count=1,
timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 1300,
'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0,32], [222,871], [1257,1300]],
'timestamp': 3583725976.97}}
self.driver = MflmADCPSDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
from mi.idk.dataset.unit_test import DataSetQualificationTestCase
from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys
from mi.dataset.dataset_driver import DriverParameter, DriverStateKey
from mi.dataset.driver.mflm.adcp.driver import MflmADCPSDataSetDriver, DataSourceKey
from mi.dataset.parser.adcps import AdcpsParserDataParticle
TELEM_DIR = '/tmp/dsatest'
# Fill in the blanks to initialize data set test case
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.adcp.driver',
driver_class='MflmADCPSDataSetDriver',
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = MflmADCPSDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.RESOURCE_ID: 'adcps',
DataSourceConfigKey.HARVESTER:
{
DataSourceKey.ADCPS_JLN_SIO_MULE: {
DataSetDriverConfigKeys.DIRECTORY: TELEM_DIR,
DataSetDriverConfigKeys.PATTERN: 'node59p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 5
}
},
DataSourceConfigKey.PARSER: { DataSourceKey.ADCPS_JLN_SIO_MULE: {} }
}
)
from mi.dataset.dataset_driver import DriverParameter, DriverStateKey
from mi.dataset.driver.mflm.adcp.driver import MflmADCPSDataSetDriver
from mi.dataset.parser.adcps import AdcpsParserDataParticle
from pyon.agent.agent import ResourceAgentState
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
# Fill in the blanks to initialize data set test case
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.adcp.driver',
driver_class='MflmADCPSDataSetDriver',
agent_resource_id='123xyz',
agent_name='Agent007',
agent_packet_config=MflmADCPSDataSetDriver.stream_config(),
startup_config={
DataSourceConfigKey.RESOURCE_ID: 'adcps',
DataSourceConfigKey.HARVESTER: {
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: 'node59p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
})
SAMPLE_STREAM = 'adcps_parsed'
###############################################################################
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_1.txt.result.yml',
count=1, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, count=2, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, count=1, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 1300,
'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0,32], [222,871], [1257,1300]],
'timestamp': 3583725976.97}}
self.driver = MflmADCPSDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(AdcpsParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_1-2.txt.result.yml',
count=2, timeout=10)
# This file has had a section of AD data replaced with 0s, this should start a new
# sequence for the data following the missing AD data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_3.txt.result.yml',
count=1, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_4.txt.result.yml',
count=1, timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers, [0-32], [222-871], [1833-2000]
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, 'test_data_1-4.txt.result.yml',
count=4, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_1.txt.result.yml',
count=1,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_2.txt.result.yml',
count=1,
timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, count=2, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle, count=1, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {
'last_filesize': 1300,
'last_checksum': 'e56e28e6bd67c6b00c6702c9f9a13f93'
},
DataSourceConfigKey.PARSER: {
'in_process_data': [],
'unprocessed_data': [[0, 32], [222, 871], [1257, 1300]],
'timestamp': 3583725976.97
}
}
self.driver = MflmADCPSDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(AdcpsParserDataParticle,
'test_data_2.txt.result.yml',
count=1,
timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_1-2.txt.result.yml',
count=2,
timeout=10)
# This file has had a section of AD data replaced with 0s, this should start a new
# sequence for the data following the missing AD data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_3.txt.result.yml',
count=1,
timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_4.txt.result.yml',
count=1,
timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers, [0-32], [222-871], [1833-2000]
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(AdcpsParserDataParticle,
'test_data_1-4.txt.result.yml',
count=4,
timeout=10)
