def test_midstate_start(self): """ Test the ability to stop and restart the process """ recov_filename = 'C0000038.DAT' mule_filename = 'node58p1.dat' log.debug('CAG MIDSTATE START set memento') memento = { DataTypeKey.CTDPF_CKL_WFP: { recov_filename: { DriverStateKey.FILE_SIZE: 4012, DriverStateKey.FILE_CHECKSUM: '5d305d5b326baf5e01efe387b99b2285', DriverStateKey.FILE_MOD_DATE: '1405695548.0562634', DriverStateKey.INGESTED: False, DriverStateKey.PARSER_STATE: { 'position': 66, 'records_read': 6, 'metadata_sent': True } } }, DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: { mule_filename: { DriverStateKey.FILE_SIZE: 9866, DriverStateKey.FILE_CHECKSUM: '86584e1b27baa938c920495f7ed4d000', # Bogus checksum (intentional) DriverStateKey.FILE_MOD_DATE: '1404933794.0', DriverStateKey.PARSER_STATE: { StateKey.IN_PROCESS_DATA: [[2982, 3189, 15, 14]], StateKey.UNPROCESSED_DATA: [[2982, 3189], [4058, 4059], [4673, 5540], [7423, 7424], [8730, 9828]], StateKey.FILE_SIZE: 9866 } } } } log.debug('CAG MIDSTATE START initial driver config is %s', self._driver_config()['startup_config']) log.debug('CAG MIDSTATE START build driver') driver = CtdpfCklWfpDataSetDriver( self._driver_config()['startup_config'], memento, self.data_callback, self.state_callback, self.event_callback, self.exception_callback) #************************** RECOVERED ************************** self.clear_async_data() log.debug('CAG MIDSTATE START file load') self.create_sample_data_set_dir('BIG_C0000038.dat', RECOV_DATA_DIR, recov_filename) self.create_sample_data_set_dir('NORMAL_node58p1.dat', MULE_DATA_DIR, mule_filename) log.debug('CAG MIDSTATE START start recovered sampling') driver.start_sampling() log.debug('CAG MIDSTATE START check the results') self.assert_data(RECOV_PARTICLES, 'subBIG_C0000038.yml', count=357, timeout=200) self.assert_data(MULE_PARTICLES, 'Last_172_NORMAL_node58p1.yml', count=172, timeout=100) # self.assert_data(MULE_PARTICLES, 'NORMAL_node58p1.yml', count=186, timeout=100) log.debug('CAG MIDSTATE START stop sampling') self.driver.stop_sampling()
MULE_PARTICLES = (CtdpfCklWfpSioMuleDataParticle, CtdpfCklWfpSioMuleMetadataParticle) RECOV_PARTICLES = (CtdpfCklWfpDataParticle, CtdpfCklWfpMetadataParticle) RECOV_DATA_DIR = '/tmp/recov_dsatest' MULE_DATA_DIR = '/tmp/mule_dsatest' # Driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.ctdpf_ckl.wfp_sio_mule.driver', driver_class='CtdpfCklWfpDataSetDriver', agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=CtdpfCklWfpDataSetDriver.stream_config(), startup_config = { DataSourceConfigKey.RESOURCE_ID: 'ctdpf_ckl_wfp_sio_mule', DataSourceConfigKey.HARVESTER: { DataTypeKey.CTDPF_CKL_WFP: { DataSetDriverConfigKeys.DIRECTORY: RECOV_DATA_DIR, DataSetDriverConfigKeys.PATTERN: 'C*.DAT', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: { DataSetDriverConfigKeys.DIRECTORY: MULE_DATA_DIR, DataSetDriverConfigKeys.PATTERN: 'node58p1.dat', DataSetDriverConfigKeys.FREQUENCY: 1,
from mi.dataset.parser.sio_mule_common import StateKey MULE_PARTICLES = (CtdpfCklWfpSioMuleDataParticle, CtdpfCklWfpSioMuleMetadataParticle) RECOV_PARTICLES = (CtdpfCklWfpDataParticle, CtdpfCklWfpMetadataParticle) RECOV_DATA_DIR = '/tmp/recov_dsatest' MULE_DATA_DIR = '/tmp/mule_dsatest' # Driver details DataSetTestCase.initialize( driver_module='mi.dataset.driver.ctdpf_ckl.wfp_sio_mule.driver', driver_class='CtdpfCklWfpDataSetDriver', agent_resource_id='123xyz', agent_name='Agent007', agent_packet_config=CtdpfCklWfpDataSetDriver.stream_config(), startup_config={ DataSourceConfigKey.RESOURCE_ID: 'ctdpf_ckl_wfp_sio_mule', DataSourceConfigKey.HARVESTER: { DataTypeKey.CTDPF_CKL_WFP: { DataSetDriverConfigKeys.DIRECTORY: RECOV_DATA_DIR, DataSetDriverConfigKeys.PATTERN: 'C*.DAT', DataSetDriverConfigKeys.FREQUENCY: 1, }, DataTypeKey.CTDPF_CKL_WFP_SIO_MULE: { DataSetDriverConfigKeys.DIRECTORY: MULE_DATA_DIR, DataSetDriverConfigKeys.PATTERN: 'node58p1.dat', DataSetDriverConfigKeys.FREQUENCY: 1, } }, DataSourceConfigKey.PARSER: {