def test_file_mod_wait_time(self):
"""
Test that the harvester waits the required amount of time before finding
files that are being modified
"""
# just touch the file to create it
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
# keep track of how long it takes to find the file approximately
file_found_time = 0;
while(not self.data_found):
time.sleep(1)
file_found_time += 1
if file_found_time > 60:
raise Exception("Timeout waiting to find file")
if file_found_time < CONFIG.get(DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
# we found the file before the mod time, this is bad!
file_harvester.shutdown()
self.fail('Files found in %s seconds' % file_found_time)
log.debug('File found in %s seconds', file_found_time)
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 1)
# Wait for three lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
memento = self.get_file_metadata(FILENAME)
log.debug('Starting with memento %s', memento)
self.starting_lines = 2
# start the harvester at the end of the current data
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# just touch the file so it is created
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
memento = self.get_file_metadata(FILENAME)
log.debug('Starting with memento %s', memento)
self.starting_lines = 2
# start the harvester at the end of the current data
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)
# start the harvester at the end of the current data
self.current_offset = 32
file_offset = 32
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 2)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find the file changes from a file already in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
self.wait_for_data(2)
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_multi_line(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 4, .9)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_multi_line(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 4, .9)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_file_mod_wait_time(self):
"""
Test that the harvester waits the required amount of time before finding
files that are being modified
"""
# just touch the file to create it
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
# keep track of how long it takes to find the file approximately
file_found_time = 0
while (not self.data_found):
time.sleep(1)
file_found_time += 1
if file_found_time > 60:
raise Exception("Timeout waiting to find file")
if file_found_time < CONFIG.get(
DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
# we found the file before the mod time, this is bad!
file_harvester.shutdown()
self.fail('Files found in %s seconds' % file_found_time)
log.debug('File found in %s seconds', file_found_time)
file_harvester.shutdown()
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# just touch the file so it is created
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find the file changes from a file already in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
self.wait_for_data(2)
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()