def test_midstate_start(self):
"""
Test the ability to stop and restart the process
"""
recov_filename = 'C0000038.DAT'
mule_filename = 'node58p1.dat'
log.debug('CAG MIDSTATE START set memento')
memento = {
DataTypeKey.CTDPF_CKL_WFP: {
recov_filename: {
DriverStateKey.FILE_SIZE: 4012,
DriverStateKey.FILE_CHECKSUM: '5d305d5b326baf5e01efe387b99b2285',
DriverStateKey.FILE_MOD_DATE: '1405695548.0562634',
DriverStateKey.INGESTED: False,
DriverStateKey.PARSER_STATE: {
'position': 66,
'records_read': 6,
'metadata_sent': True
}
}
},
DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: {
mule_filename: {
DriverStateKey.FILE_SIZE: 9866,
DriverStateKey.FILE_CHECKSUM: '86584e1b27baa938c920495f7ed4d000', # Bogus checksum (intentional)
DriverStateKey.FILE_MOD_DATE: '1404933794.0',
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [[2982, 3189, 15, 14]],
StateKey.UNPROCESSED_DATA: [[2982, 3189],
[4058, 4059],
[4673, 5540],
[7423, 7424],
[8730, 9828]],
StateKey.FILE_SIZE: 9866
}
}
}
}
log.debug('CAG MIDSTATE START initial driver config is %s', self._driver_config()['startup_config'])
log.debug('CAG MIDSTATE START build driver')
driver = CtdpfCklWfpDataSetDriver(
self._driver_config()['startup_config'],
memento,
self.data_callback,
self.state_callback,
self.event_callback,
self.exception_callback)
#************************** RECOVERED **************************
self.clear_async_data()
log.debug('CAG MIDSTATE START file load')
self.create_sample_data_set_dir('BIG_C0000038.dat', RECOV_DATA_DIR, recov_filename)
self.create_sample_data_set_dir('NORMAL_node58p1.dat', MULE_DATA_DIR, mule_filename)
log.debug('CAG MIDSTATE START start recovered sampling')
driver.start_sampling()
log.debug('CAG MIDSTATE START check the results')
self.assert_data(RECOV_PARTICLES, 'subBIG_C0000038.yml', count=357, timeout=200)
self.assert_data(MULE_PARTICLES, 'Last_172_NORMAL_node58p1.yml', count=172, timeout=100)
# self.assert_data(MULE_PARTICLES, 'NORMAL_node58p1.yml', count=186, timeout=100)
log.debug('CAG MIDSTATE START stop sampling')
self.driver.stop_sampling()
def test_midstate_start(self):
"""
Test the ability to stop and restart the process
"""
recov_filename = 'C0000038.DAT'
mule_filename = 'node58p1.dat'
log.debug('CAG MIDSTATE START set memento')
memento = {
DataTypeKey.CTDPF_CKL_WFP: {
recov_filename: {
DriverStateKey.FILE_SIZE: 4012,
DriverStateKey.FILE_CHECKSUM:
'5d305d5b326baf5e01efe387b99b2285',
DriverStateKey.FILE_MOD_DATE: '1405695548.0562634',
DriverStateKey.INGESTED: False,
DriverStateKey.PARSER_STATE: {
'position': 66,
'records_read': 6,
'metadata_sent': True
}
}
},
DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: {
mule_filename: {
DriverStateKey.FILE_SIZE: 9866,
DriverStateKey.FILE_CHECKSUM:
'86584e1b27baa938c920495f7ed4d000', # Bogus checksum (intentional)
DriverStateKey.FILE_MOD_DATE: '1404933794.0',
DriverStateKey.PARSER_STATE: {
StateKey.IN_PROCESS_DATA: [[2982, 3189, 15, 14]],
StateKey.UNPROCESSED_DATA: [[2982, 3189], [4058, 4059],
[4673, 5540], [7423, 7424],
[8730, 9828]],
StateKey.FILE_SIZE:
9866
}
}
}
}
log.debug('CAG MIDSTATE START initial driver config is %s',
self._driver_config()['startup_config'])
log.debug('CAG MIDSTATE START build driver')
driver = CtdpfCklWfpDataSetDriver(
self._driver_config()['startup_config'], memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
#************************** RECOVERED **************************
self.clear_async_data()
log.debug('CAG MIDSTATE START file load')
self.create_sample_data_set_dir('BIG_C0000038.dat', RECOV_DATA_DIR,
recov_filename)
self.create_sample_data_set_dir('NORMAL_node58p1.dat', MULE_DATA_DIR,
mule_filename)
log.debug('CAG MIDSTATE START start recovered sampling')
driver.start_sampling()
log.debug('CAG MIDSTATE START check the results')
self.assert_data(RECOV_PARTICLES,
'subBIG_C0000038.yml',
count=357,
timeout=200)
self.assert_data(MULE_PARTICLES,
'Last_172_NORMAL_node58p1.yml',
count=172,
timeout=100)
# self.assert_data(MULE_PARTICLES, 'NORMAL_node58p1.yml', count=186, timeout=100)
log.debug('CAG MIDSTATE START stop sampling')
self.driver.stop_sampling()
MULE_PARTICLES = (CtdpfCklWfpSioMuleDataParticle, CtdpfCklWfpSioMuleMetadataParticle)
RECOV_PARTICLES = (CtdpfCklWfpDataParticle, CtdpfCklWfpMetadataParticle)
RECOV_DATA_DIR = '/tmp/recov_dsatest'
MULE_DATA_DIR = '/tmp/mule_dsatest'
# Driver details
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.ctdpf_ckl.wfp_sio_mule.driver',
driver_class='CtdpfCklWfpDataSetDriver',
agent_resource_id='123xyz',
agent_name='Agent007',
agent_packet_config=CtdpfCklWfpDataSetDriver.stream_config(),
startup_config = {
DataSourceConfigKey.RESOURCE_ID: 'ctdpf_ckl_wfp_sio_mule',
DataSourceConfigKey.HARVESTER:
{
DataTypeKey.CTDPF_CKL_WFP:
{
DataSetDriverConfigKeys.DIRECTORY: RECOV_DATA_DIR,
DataSetDriverConfigKeys.PATTERN: 'C*.DAT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataTypeKey.CTDPF_CKL_WFP_SIO_MULE:
{
DataSetDriverConfigKeys.DIRECTORY: MULE_DATA_DIR,
DataSetDriverConfigKeys.PATTERN: 'node58p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
from mi.dataset.parser.sio_mule_common import StateKey
MULE_PARTICLES = (CtdpfCklWfpSioMuleDataParticle,
CtdpfCklWfpSioMuleMetadataParticle)
RECOV_PARTICLES = (CtdpfCklWfpDataParticle, CtdpfCklWfpMetadataParticle)
RECOV_DATA_DIR = '/tmp/recov_dsatest'
MULE_DATA_DIR = '/tmp/mule_dsatest'
# Driver details
DataSetTestCase.initialize(
driver_module='mi.dataset.driver.ctdpf_ckl.wfp_sio_mule.driver',
driver_class='CtdpfCklWfpDataSetDriver',
agent_resource_id='123xyz',
agent_name='Agent007',
agent_packet_config=CtdpfCklWfpDataSetDriver.stream_config(),
startup_config={
DataSourceConfigKey.RESOURCE_ID: 'ctdpf_ckl_wfp_sio_mule',
DataSourceConfigKey.HARVESTER: {
DataTypeKey.CTDPF_CKL_WFP: {
DataSetDriverConfigKeys.DIRECTORY: RECOV_DATA_DIR,
DataSetDriverConfigKeys.PATTERN: 'C*.DAT',
DataSetDriverConfigKeys.FREQUENCY: 1,
},
DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: {
DataSetDriverConfigKeys.DIRECTORY: MULE_DATA_DIR,
DataSetDriverConfigKeys.PATTERN: 'node58p1.dat',
DataSetDriverConfigKeys.FREQUENCY: 1,
}
},
DataSourceConfigKey.PARSER: {
