def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {
'last_filesize': 500,
'last_checksum': '36cd31b3ae1c90eb5b1a0ed7d7be6512'
},
DataSourceConfigKey.PARSER: {
'in_process_data': [],
'unprocessed_data': [[0, 12], [336, 394], [467, 500]],
'timestamp': 3583612801.0
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=12,
timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {"last_filesize": 893, "last_checksum": "b859e40320ac396a5991d80a655bc161"},
DataSourceConfigKey.PARSER: {
"in_process_data": [],
"unprocessed_data": [[0, 50], [374, 432], [892, 893]],
"timestamp": 3583656001.0,
},
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()["startup_config"],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback,
)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, "test_data_2.txt.result.yml", count=12, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 6000,
'last_checksum': 'aa1cc1aa816e99e11d8e88fc56f887e7'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0, 12], [336, 394], [467, 2010], [5544, 6000]],
'timestamp': 3583634401.0}}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DriverStateKey.FILE_SIZE: 6000,
DriverStateKey.FILE_CHECKSUM: 'aa1cc1aa816e99e11d8e88fc56f887e7',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
'in_process_data': [],
'unprocessed_data': [[0, 12], [336, 394], [467, 2010],
[5544, 6000]],
'timestamp':
3583634401.0
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=2,
timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester'][DataSourceKey.CTDMO_GHQR_SIO_MULE]['directory'],
driver_config['harvester'][DataSourceKey.CTDMO_GHQR_SIO_MULE]['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.CTDMO_GHQR_SIO_MULE: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 6000,
DriverStateKey.FILE_CHECKSUM: 'aa1cc1aa816e99e11d8e88fc56f887e7',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [],
StateKey.UNPROCESSED_DATA:[[0, 12], [336, 394], [5924,6000]],
StateKey.FILE_SIZE: 6000
}
}
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {DriverStateKey.FILE_SIZE: 6000,
DriverStateKey.FILE_CHECKSUM: 'aa1cc1aa816e99e11d8e88fc56f887e7',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {'in_process_data': [],
'unprocessed_data':[[0, 12], [336, 394], [467, 2010], [5544, 6000]],
'timestamp': 3583634401.0
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def test_get(self):
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat")
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.assert_data(CtdmoParserDataParticle, 'test_data_1.txt.result.yml',
count=4, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step4.dat", TELEM_DIR, "node59p1.dat")
self.assert_data((CtdmoParserDataParticle, CtdmoOffsetDataParticle),
count=4, timeout=10)
self.driver.stop_sampling()
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
# create the file so that it is unreadable
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester'][DataSourceKey.CTDMO_GHQR_SIO_MULE]['directory'],
driver_config['harvester'][DataSourceKey.CTDMO_GHQR_SIO_MULE]['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DataSourceKey.CTDMO_GHQR_SIO_MULE: {
"node59p1.dat": {
DriverStateKey.FILE_SIZE: 6000,
DriverStateKey.FILE_CHECKSUM: 'aa1cc1aa816e99e11d8e88fc56f887e7',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
'in_process_data': [],
'unprocessed_data':[[0, 12], [336, 394], [5924,6000]],
}
}
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data_set_dir("node59p1_step2.dat", TELEM_DIR, "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
def test_back_fill(self):
"""
Test refilled blocks are sent correctly. There is only one file
that just has data appended or inserted into it, or if there is missing
data can be added back later
"""
self.create_sample_data_set_dir("node59p1_step1.dat", TELEM_DIR, "node59p1.dat")
self.driver.start_sampling()
# step 1 contains 4 blocks, start with this and get both since we used them
# separately in other tests
self.clear_async_data()
self.assert_data(CtdmoParserDataParticle, 'test_data_1.txt.result.yml',
count=4, timeout=10)
# This file has had a section of CT data replaced with 0s
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step3.dat', TELEM_DIR, "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_3.txt.result.yml',
count=2, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data
self.clear_async_data()
self.create_sample_data_set_dir('node59p1_step4.dat', TELEM_DIR, "node59p1.dat")
self.assert_data((CtdmoParserDataParticle, CtdmoOffsetDataParticle),
'test_data_4.txt.result.yml', count=4, timeout=10)
from mi.idk.dataset.unit_test import DataSetIntegrationTestCase
from mi.idk.dataset.unit_test import DataSetQualificationTestCase
from mi.dataset.dataset_driver import DataSourceConfigKey, DataSetDriverConfigKeys
from mi.dataset.dataset_driver import DriverParameter, DriverStateKey
from mi.dataset.driver.mflm.ctd.driver import MflmCTDMODataSetDriver, DataSourceKey
from mi.dataset.parser.ctdmo import CtdmoParserDataParticle, DataParticleType
from mi.dataset.parser.ctdmo import CtdmoOffsetDataParticle
TELEM_DIR = '/tmp/dsatest'
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.ctd.driver',
driver_class="MflmCTDMODataSetDriver",
agent_resource_id = '123xyz',
agent_name = 'Agent007',
agent_packet_config = MflmCTDMODataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.HARVESTER:
{
DataSourceKey.CTDMO_GHQR_SIO_MULE:
{
DataSetDriverConfigKeys.DIRECTORY: TELEM_DIR,
DataSetDriverConfigKeys.PATTERN: 'node59p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 30,
}
},
DataSourceConfigKey.PARSER: {
DataSourceKey.CTDMO_GHQR_SIO_MULE: {'inductive_id': 55}
}
}
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()["startup_config"]
log.debug("startup config %s", driver_config)
fullfile = os.path.join(driver_config["harvester"]["directory"], driver_config["harvester"]["pattern"])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_1.txt.result.yml", count=3, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_2.txt.result.yml", count=12, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=24, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=3, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {"last_filesize": 893, "last_checksum": "b859e40320ac396a5991d80a655bc161"},
DataSourceConfigKey.PARSER: {
"in_process_data": [],
"unprocessed_data": [[0, 50], [374, 432], [892, 893]],
"timestamp": 3583656001.0,
},
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()["startup_config"],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback,
)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, "test_data_2.txt.result.yml", count=12, timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_1-2.txt.result.yml", count=15, timeout=10)
# This file has had a section of CT data replaced with 0s, this should start a new
# sequence for the data following the missing CT data
self.clear_async_data()
self.create_sample_data("node59p1_step3.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_3.txt.result.yml", count=12, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_4.txt.result.yml", count=12, timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file (there are some sections of bad data that don't
# match in headers, [0-50], [374-432], [1197-1471]
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, "test_data_1-4.txt.result.yml", count=39, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_1.txt.result.yml',
count=2, timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=2, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=2, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {DataSourceConfigKey.HARVESTER: {'last_filesize': 6000,
'last_checksum': 'aa1cc1aa816e99e11d8e88fc56f887e7'},
DataSourceConfigKey.PARSER: {'in_process_data': [],
'unprocessed_data':[[0, 12], [336, 394], [467, 2010], [5544, 6000]],
'timestamp': 3583634401.0}}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'],
self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle, 'test_data_2.txt.result.yml',
count=2, timeout=10)
def test_back_fill(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_1.txt.result.yml',
count=2, timeout=10)
# This file has had a section of CT data replaced with 0s, this should start a new
# sequence for the data following the missing CT data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_3.txt.result.yml',
count=1, timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, 'test_data_4.txt.result.yml',
count=3, timeout=10)
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.assert_data(CtdmoParserDataParticle,
'test_data_1.txt.result.yml',
count=2,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=2,
timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=2, timeout=10)
self.driver.stop_sampling()
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(ValueError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
driver_config = self._driver_config()['startup_config']
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
mod_time = os.path.getmtime(fullfile)
# Create and store the new driver state
self.memento = {
DriverStateKey.FILE_SIZE: 6000,
DriverStateKey.FILE_CHECKSUM: 'aa1cc1aa816e99e11d8e88fc56f887e7',
DriverStateKey.FILE_MOD_DATE: mod_time,
DriverStateKey.PARSER_STATE: {
'in_process_data': [],
'unprocessed_data': [[0, 12], [336, 394], [467, 2010],
[5544, 6000]],
'timestamp':
3583634401.0
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=2,
timeout=10)
def test_back_fill(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.driver.start_sampling()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.assert_data(CtdmoParserDataParticle,
'test_data_1.txt.result.yml',
count=2,
timeout=10)
# This file has had a section of CT data replaced with 0s, this should start a new
# sequence for the data following the missing CT data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_3.txt.result.yml',
count=1,
timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_4.txt.result.yml',
count=3,
timeout=10)
from mi.dataset.driver.mflm.ctd.driver import MflmCTDMODataSetDriver
from mi.dataset.parser.ctdmo import CtdmoParserDataParticle
from pyon.agent.agent import ResourceAgentState
from interface.objects import CapabilityType
from interface.objects import AgentCapability
from interface.objects import ResourceAgentErrorEvent
from interface.objects import ResourceAgentConnectionLostErrorEvent
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.mflm.ctd.driver',
driver_class="MflmCTDMODataSetDriver",
agent_resource_id='123xyz',
agent_name='Agent007',
agent_packet_config=MflmCTDMODataSetDriver.stream_config(),
startup_config={
DataSourceConfigKey.HARVESTER: {
DataSetDriverConfigKeys.DIRECTORY: '/tmp/dsatest',
DataSetDriverConfigKeys.STORAGE_DIRECTORY: '/tmp/stored_dsatest',
DataSetDriverConfigKeys.PATTERN: 'node59p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME: 30,
},
DataSourceConfigKey.PARSER: {
'inductive_id': 55
}
})
SAMPLE_STREAM = 'ctdmo_parsed'
class IntegrationTest(DataSetIntegrationTestCase):
def clean_file(self):
# remove just the file we are using
driver_config = self._driver_config()['startup_config']
log.debug('startup config %s', driver_config)
fullfile = os.path.join(driver_config['harvester']['directory'],
driver_config['harvester']['pattern'])
if os.path.exists(fullfile):
os.remove(fullfile)
def test_get(self):
self.clean_file()
# Start sampling and watch for an exception
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_1.txt.result.yml',
count=3,
timeout=10)
# there is only one file we read from, this example 'appends' data to
# the end of the node59p1.dat file, and the data from the new append
# is returned (not including the original data from _step1)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=12,
timeout=10)
# now 'appends' the rest of the data and just check if we get the right number
self.clear_async_data()
self.create_sample_data("node59p1_step4.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=24, timeout=10)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data("node59p1_step1.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle, count=3, timeout=10)
def test_harvester_new_file_exception(self):
"""
Test an exception raised after the driver is started during
the file read. Should call the exception callback.
"""
self.clean_file()
# create the file so that it is unreadable
self.create_sample_data("node59p1_step1.dat", "node59p1.dat", mode=000)
# Start sampling and watch for an exception
self.driver.start_sampling()
self.assert_exception(IOError)
# At this point the harvester thread is dead. The agent
# exception handle should handle this case.
def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
self.clean_file()
# Create and store the new driver state
self.memento = {
DataSourceConfigKey.HARVESTER: {
'last_filesize': 500,
'last_checksum': '36cd31b3ae1c90eb5b1a0ed7d7be6512'
},
DataSourceConfigKey.PARSER: {
'in_process_data': [],
'unprocessed_data': [[0, 12], [336, 394], [467, 500]],
'timestamp': 3583612801.0
}
}
self.driver = MflmCTDMODataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdmoParserDataParticle,
'test_data_2.txt.result.yml',
count=12,
timeout=10)
def test_sequences(self):
"""
Test new sequence flags are set correctly. There is only one file
that just has data appended or inserted into it, so new sequences
can occur in both cases, or if there is missing data in between two sequences
"""
self.clean_file()
self.driver.start_sampling()
self.clear_async_data()
# step 2 contains 2 blocks, start with this and get both since we used them
# separately in other tests (no new sequences)
self.clear_async_data()
self.create_sample_data("node59p1_step2.dat", "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_1-2.txt.result.yml',
count=15,
timeout=10)
# This file has had a section of CT data replaced with 0s, this should start a new
# sequence for the data following the missing CT data
self.clear_async_data()
self.create_sample_data('node59p1_step3.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_3.txt.result.yml',
count=12,
timeout=10)
# Now fill in the zeroed section from step3, this should just return the new
# data with a new sequence flag
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_4.txt.result.yml',
count=12,
timeout=10)
# start over now, using step 4, make sure sequence flags just account for
# missing data in file
# note that node59p1_step4.dat has been doctored from the original file to
# take out one data section in between two CT packets (3rd and 4th) so there
# could be two continuous packets (no new seq)
self.driver.stop_sampling()
# reset the parser and harvester states
self.driver.clear_states()
self.driver.start_sampling()
self.clear_async_data()
self.create_sample_data('node59p1_step4.dat', "node59p1.dat")
self.assert_data(CtdmoParserDataParticle,
'test_data_1-4.txt.result.yml',
count=39,
timeout=10)
