def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 314,
'last_checksum': '515e5da08a6b4bb0d197e62c410da532'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0,69]],
'timestamp': 3583581301.0}}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle,
'test_data_3.txt.result.yml',
count=3)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle,
'test_data_4.txt.result.yml',
count=1)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-4.txt.result.yml',
count=7)
def test_sequences(self):
"""
Test new sequence flags are set correctly
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml',
count=2, timeout=10)
# This file has had a section of FL data replaced with 0s, this should start a new
# sequence for the data following the missing AD data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
count=3, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
count=1, timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml',
count=6, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
driver_config = self._driver_config()['startup_config']
dosta_telem_config = driver_config['harvester'][
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
fullfile = os.path.join(dosta_telem_config['directory'],
dosta_telem_config['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM:
'515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [],
StateKey.UNPROCESSED_DATA: [[0, 69]],
StateKey.FILE_SIZE: 314
}
}
}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle,
'test_data_2.txt.result.yml',
count=1)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 314,
'last_checksum': '515e5da08a6b4bb0d197e62c410da532'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0,69]],
'timestamp': 3583581301.0}}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml',
count=1, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=4, timeout=10)
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=1, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {'in_process_data': [],
'unprocessed_data':[[0,69]],
'timestamp': 3583581301.0}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml', count=3)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
count=3)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
count=1)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-4.txt.result.yml', count=7)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
driver_config = self._driver_config()['startup_config']
dosta_telem_config = driver_config['harvester'][DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
fullfile = os.path.join(dosta_telem_config['directory'], dosta_telem_config['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [],
StateKey.UNPROCESSED_DATA:[[0,69]],
StateKey.FILE_SIZE: 314
}
}
}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1)
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml',
count=1, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=4, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=1, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 314,
'last_checksum': '515e5da08a6b4bb0d197e62c410da532'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0,69]],
'timestamp': 3583581301.0}}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml',
count=2, timeout=10)
# This file has had a section of FL data replaced with 0s, this should start a new
# sequence for the data following the missing AD data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
count=3, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
count=1, timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml',
count=6, timeout=10)
from mi.dataset.dataset_driver import DriverParameter
from mi.dataset.driver.mflm.dosta.driver import MflmDOSTADDataSetDriver
from mi.dataset.parser.dostad import DostadParserDataParticle
from pyon.agent.agent import ResourceAgentState
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.dosta.driver',
driver_class='MflmDOSTADDataSetDriver',
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = MflmDOSTADDataSetDriver.stream_config(),
startup_config = {
'harvester':
{
'directory': '/tmp/dsatest',
'pattern': 'node59p1.dat',
'frequency': 1,
},
'parser': {}
}
)
SAMPLE_STREAM = 'dostad_parsed'
###############################################################################
# INTEGRATION TESTS #
class IntegrationTest(DataSetIntegrationTestCase):
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1.txt.result.yml',
count=2,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle,
'test_data_2.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step4.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, count=4)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
mode=000,
copy_metadata=False)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
driver_config = self._driver_config()['startup_config']
dosta_telem_config = driver_config['harvester'][
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
fullfile = os.path.join(dosta_telem_config['directory'],
dosta_telem_config['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM:
'515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [],
StateKey.UNPROCESSED_DATA: [[0, 69]],
StateKey.FILE_SIZE: 314
}
}
}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle,
'test_data_2.txt.result.yml',
count=1)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle,
'test_data_3.txt.result.yml',
count=3)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle,
'test_data_4.txt.result.yml',
count=1)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-4.txt.result.yml',
count=7)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data(DostadParserDataParticle,
'test_data_all_good.txt.result.yml',
count=1)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data(
(DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data(DostadParserDataParticle,
'test_data_all_good.txt.result.yml',
count=1)
class IntegrationTest(DataSetIntegrationTestCase):
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1.txt.result.yml', count=2, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step4.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, count=4)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
mode=000, copy_metadata=False)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
driver_config = self._driver_config()['startup_config']
dosta_telem_config = driver_config['harvester'][DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED]
fullfile = os.path.join(dosta_telem_config['directory'], dosta_telem_config['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.DOSTA_ABCDJM_SIO_TELEMETERED: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
'in_process_data': [],
'unprocessed_data':[[0,69]]
}
}
}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml', count=3)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
count=3)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
count=1)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-4.txt.result.yml', count=7)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat")
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml', count=3)
self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml',
count=1)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR, "node59p1.dat")
self.assert_data((DostadParserDataParticle, DostadMetadataDataParticle),
'test_data_1-2.txt.result.yml', count=3)
self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR, "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_all_good.txt.result.yml',
count=1)
class IntegrationTest(DataSetIntegrationTestCase):
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir(
"node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data((DostadParserTelemeteredDataParticle, DostadParserTelemeteredMetadataDataParticle),
'test_data_1.txt.result.yml', count=2, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir(
"node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(DostadParserTelemeteredDataParticle, 'test_data_2.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir(
"node59p1_step4.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(DostadParserTelemeteredDataParticle, count=4)
def test_get_recovered(self):
# RECOVERED VERSIONS appear with _recovered appended
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir(
"DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False
)
self.assert_data(
(DostadParserRecoveredDataParticle, DostadParserRecoveredMetadataDataParticle),
'test_data_1r.txt.result.yml',
count=2,
timeout=10
)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir(
"DOS15908_1st7_step2.DAT",
RECOV_DIR,
"DOS15909.DAT",
copy_metadata=False
)
self.assert_data(DostadParserRecoveredDataParticle,
'test_data_2r.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir(
"DOS15908.DAT",
RECOV_DIR,
"DOS15910.DAT",
copy_metadata=False
)
self.assert_data(DostadParserRecoveredDataParticle, count=4)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir(
"node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
mode=000,
copy_metadata=False
)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_harvester_new_file_exception_recovered(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir(
"DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15909.DAT",
mode=000
)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir(
"node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
# create the recovered file
self.create_sample_data_set_dir(
"DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False
)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(
DostadParserTelemeteredDataParticle,
'test_data_1_ss1.txt.result.yml',
count=1,
timeout=10
)
self.assert_data(
DostadParserRecoveredDataParticle,
'test_data_1r_ss1.txt.result.yml',
count=1,
timeout=10
)
self.driver.stop_sampling()
self.driver.start_sampling()
self.assert_data(
DostadParserTelemeteredMetadataDataParticle,
'test_data_1_ss2.txt.result.yml',
count=1,
timeout=10
)
self.assert_data(
DostadParserRecoveredMetadataDataParticle,
'test_data_1r_ss2.txt.result.yml',
count=1,
timeout=10
)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir(
"node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(
(DostadParserTelemeteredDataParticle, DostadParserTelemeteredMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3
)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir(
'node59p1_step3.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(
DostadParserTelemeteredDataParticle,
'test_data_3.txt.result.yml',
count=3
)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir(
'node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(
DostadParserTelemeteredDataParticle,
'test_data_4.txt.result.yml',
count=1
)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir(
'node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False
)
self.assert_data(
(DostadParserTelemeteredDataParticle, DostadParserTelemeteredMetadataDataParticle),
'test_data_1-4.txt.result.yml',
count=7
)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir(
"node59p1_all_good1.dat",
TELEM_DIR,
"node59p1.dat"
)
self.assert_data(
(DostadParserTelemeteredDataParticle, DostadParserTelemeteredMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3
)
self.create_sample_data_set_dir(
"node59p1_all_good.dat",
TELEM_DIR,
"node59p1.dat"
)
self.assert_data(
DostadParserTelemeteredDataParticle,
'test_data_all_good.txt.result.yml',
count=1
)
def test_all_good_recovered(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
# Start sampling
self.driver.start_sampling()
self.create_sample_data_set_dir(
"DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False
)
self.assert_data(
(DostadParserRecoveredDataParticle, DostadParserRecoveredMetadataDataParticle),
'test_data_1r.txt.result.yml',
count=2,
timeout=10
)
self.create_sample_data_set_dir(
"DOS15908_1st7_step2.DAT",
RECOV_DIR,
"DOS15909.DAT",
copy_metadata=False
)
self.assert_data(
DostadParserRecoveredDataParticle,
'test_data_2r.txt.result.yml',
count=1
)
from mi.dataset.dataset_driver import DriverParameter, DriverStateKey
from mi.dataset.driver.mflm.dosta.driver import MflmDOSTADDataSetDriver
from mi.dataset.parser.dostad import DostadParserDataParticle
from pyon.agent.agent import ResourceAgentState
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.dosta.driver',
driver_class='MflmDOSTADDataSetDriver',
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = MflmDOSTADDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: 'node59p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataSourceConfigKey.PARSER: {}
}
)
SAMPLE_STREAM = 'dostad_parsed'
###############################################################################
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1.txt.result.yml',
count=1, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=4, timeout=10)
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, count=1, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {DriverStateKey.FILE_SIZE: 314,
DriverStateKey.FILE_CHECKSUM: '515e5da08a6b4bb0d197e62c410da532',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {'in_process_data': [],
'unprocessed_data':[[0,69]],
'timestamp': 3583581301.0}
}
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserDataParticle, 'test_data_2.txt.result.yml',
count=1, timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-2.txt.result.yml',
count=2, timeout=10)
# This file has had a section of FL data replaced with 0s, this should start a new
# sequence for the data following the missing AD data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_3.txt.result.yml',
count=3, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_4.txt.result.yml',
count=1, timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(DostadParserDataParticle, 'test_data_1-4.txt.result.yml',
count=6, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def test_get(self):
"""
Test that we can get data from files. Verify that the driver
sampling can be started and stopped
"""
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserTelemeteredDataParticle,
DostadParserTelemeteredMetadataDataParticle),
'test_data_1.txt.result.yml',
count=2,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserTelemeteredDataParticle,
'test_data_2.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step4.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserTelemeteredDataParticle, count=4)
def test_get_recovered(self):
# RECOVERED VERSIONS appear with _recovered appended
# Start sampling
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir("DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False)
self.assert_data((DostadParserRecoveredDataParticle,
DostadParserRecoveredMetadataDataParticle),
'test_data_1r.txt.result.yml',
count=2,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir("DOS15908_1st7_step2.DAT",
RECOV_DIR,
"DOS15909.DAT",
copy_metadata=False)
self.assert_data(DostadParserRecoveredDataParticle,
'test_data_2r.txt.result.yml',
count=1)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir("DOS15908.DAT",
RECOV_DIR,
"DOS15910.DAT",
copy_metadata=False)
self.assert_data(DostadParserRecoveredDataParticle, count=4)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
mode=000,
copy_metadata=False)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_harvester_new_file_exception_recovered(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir("DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15909.DAT",
mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
# create the recovered file
self.create_sample_data_set_dir("DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(DostadParserTelemeteredDataParticle,
'test_data_1_ss1.txt.result.yml',
count=1,
timeout=10)
self.assert_data(DostadParserRecoveredDataParticle,
'test_data_1r_ss1.txt.result.yml',
count=1,
timeout=10)
self.driver.stop_sampling()
self.driver.start_sampling()
self.assert_data(DostadParserTelemeteredMetadataDataParticle,
'test_data_1_ss2.txt.result.yml',
count=1,
timeout=10)
self.assert_data(DostadParserRecoveredMetadataDataParticle,
'test_data_1r_ss2.txt.result.yml',
count=1,
timeout=10)
def test_back_fill(self):
"""
Test new sequence flags are set correctly
"""
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat",
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserTelemeteredDataParticle,
DostadParserTelemeteredMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
# This file has had a section of DO data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserTelemeteredDataParticle,
'test_data_3.txt.result.yml',
count=3)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data(DostadParserTelemeteredDataParticle,
'test_data_4.txt.result.yml',
count=1)
# start over now, using step 4
self.driver.stop_sampling()
# Reset the driver with no memento
self.memento = None
self.driver = MflmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat',
TELEM_DIR,
"node59p1.dat",
copy_metadata=False)
self.assert_data((DostadParserTelemeteredDataParticle,
DostadParserTelemeteredMetadataDataParticle),
'test_data_1-4.txt.result.yml',
count=7)
def test_all_good(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
self.driver.start_sampling()
self.create_sample_data_set_dir("node59p1_all_good1.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data((DostadParserTelemeteredDataParticle,
DostadParserTelemeteredMetadataDataParticle),
'test_data_1-2.txt.result.yml',
count=3)
self.create_sample_data_set_dir("node59p1_all_good.dat", TELEM_DIR,
"node59p1.dat")
self.assert_data(DostadParserTelemeteredDataParticle,
'test_data_all_good.txt.result.yml',
count=1)
def test_all_good_recovered(self):
"""
Test that a set of data with no bad data, where there is no remaining
unprocessed data in between
"""
# Start sampling
self.driver.start_sampling()
self.create_sample_data_set_dir("DOS15908_1st7_step1.DAT",
RECOV_DIR,
"DOS15908.DAT",
copy_metadata=False)
self.assert_data((DostadParserRecoveredDataParticle,
DostadParserRecoveredMetadataDataParticle),
'test_data_1r.txt.result.yml',
count=2,
timeout=10)
self.create_sample_data_set_dir("DOS15908_1st7_step2.DAT",
RECOV_DIR,
"DOS15909.DAT",
copy_metadata=False)
self.assert_data(DostadParserRecoveredDataParticle,
'test_data_2r.txt.result.yml',
count=1)
