def test_resume_mid_file(self):
"""
Test the ability to restart the process in the middle of a file
"""
path_1 = self.create_sample_data('test_data_1.dosta.log',
"20130101.dosta.log")
path_2 = self.create_sample_data('test_data_2.dosta.log',
"20130102.dosta.log")
self.memento = {
'20130101.dosta.log': self.get_file_state(path_1, True, 190),
'20130102.dosta.log': self.get_file_state(path_2, False, 191)
}
self.memento['20130101.dosta.log']['parser_state'][
'timestamp'] = 3590524817.862
self.memento['20130102.dosta.log']['parser_state'][
'timestamp'] = 3590524819.861
self.driver = IssmCnsmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# verify data is produced
self.assert_data(Issmcnsm_dostadParserDataParticle,
'test_data_2.txt.partial-result.yml',
count=3,
timeout=10)
def test_modified(self):
"""
Test for detection of an ingested file that has been modifed after ingestion
"""
file_path = self.create_sample_data('test_data_1.dosta.log',
"20130101.dosta.log")
self.memento = {
'20130101.dosta.log': self.get_file_state(file_path, True, 190),
}
self.memento['20130101.dosta.log']['parser_state'][
'timestamp'] = 3590524817.862
self.driver = IssmCnsmDOSTADDataSetDriver(
self._driver_config()['startup_config'], self.memento,
self.data_callback, self.state_callback, self.event_callback,
self.exception_callback)
# create some data to parse
self.clear_async_data()
self.driver.start_sampling()
# overwrite the old 20130101.dosta.log file
# NOTE: this does not make you wait until file mod time, since it copies the original file
# modification time, not when you copy the file in running this test
self.create_sample_data('test_data_2.dosta.log', "20130101.dosta.log")
to = gevent.Timeout(30)
to.start()
done = False
try:
while (not done):
if 'modified_state' in self.driver._driver_state[
'20130101.dosta.log']:
log.debug(
"Found modified state %s",
self.driver._driver_state['20130101.dosta.log'].get(
'modified_state'))
done = True
if not done:
log.debug(
"modification not detected yet, sleep some more...")
gevent.sleep(5)
except Timeout:
log.error("Failed to find modified file after ingestion")
self.fail("Failed to find modified file after ingestion")
finally:
to.cancel()
