def test_stop_resume(self):
"""
Test the ability to stop and restart the process
"""
# create sample data files
self.create_sample_data('test_data_1.txt', "DATA001.TXT")
self.create_sample_data('test_data_3.txt', "DATA002.TXT")
# get file metadata for use in the state dictionary
startup_config = self._driver_config()['startup_config']
directory = startup_config[DataSourceConfigKey.HARVESTER].get(
DataSetDriverConfigKeys.DIRECTORY)
file_path_1 = os.path.join(directory, "DATA001.TXT")
# need to reset file mod time since file is created again
mod_time_1 = os.path.getmtime(file_path_1)
file_size_1 = os.path.getsize(file_path_1)
with open(file_path_1) as filehandle:
md5_checksum_1 = hashlib.md5(filehandle.read()).hexdigest()
file_path_2 = os.path.join(directory, "DATA002.TXT")
mod_time_2 = os.path.getmtime(file_path_2)
file_size_2 = os.path.getsize(file_path_2)
with open(file_path_2) as filehandle:
md5_checksum_2 = hashlib.md5(filehandle.read()).hexdigest()
# Create and store the new driver state
state = {
"DATA001.TXT": {
'ingested': True,
'file_mod_date': mod_time_1,
'file_checksum': md5_checksum_1,
'file_size': file_size_1,
'parser_state': {}
},
"DATA002.TXT": {
'ingested': False,
'file_mod_date': mod_time_2,
'file_checksum': md5_checksum_2,
'file_size': file_size_2,
'parser_state': {
'position': 201,
'timestamp': 3575062804.0
}
}
}
self.driver = WfpCTDPFKDataSetDriver(
self._driver_config()['startup_config'], state, self.data_callback,
self.state_callback, self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(CtdpfkParserDataParticle,
'test_data_3.txt.partial_results.yml',
count=5,
timeout=10)
def test_harvester_config_exception(self):
"""
Start the a driver with a bad configuration. Should raise
an exception.
"""
with self.assertRaises(ConfigurationException):
self.driver = WfpCTDPFKDataSetDriver({}, self.memento,
self.data_callback,
self.state_callback,
self.exception_callback)
