def test_harvester_without_frequency(self):
"""
Test that we can use a default frequency
"""
config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(config, memento,
self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 2)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_with_memento(self):
"""
Test that the harvester can find file as they are added to a directory,
using a memento to start partway through the indices
"""
# make sure we have 2 files already in the directory
self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'], 2, 0)
# start at index 2
dir_files = glob.glob(CONFIG['directory'] + '/' + CONFIG['pattern'])
memento = self.replace_file_index(dir_files[0], 2)
file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 3)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def _build_harvester(self, harvester_state):
self._harvester = AdditiveSequentialFileHarvester(
self._harvester_config,
harvester_state,
self._new_file_callback,
self._exception_callback
)
return self._harvester
def test_harvester_without_frequency(self):
"""
Test that we can use a default frequency
"""
config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(
config, memento, self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 2)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_with_memento(self):
"""
Test that the harvester can find file as they are added to a directory,
using a memento to start partway through the indices
"""
# make sure we have 2 files already in the directory
self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'],
2, 0)
# start at index 2
dir_files = glob.glob(CONFIG['directory'] + '/' + CONFIG['pattern'])
memento = self.replace_file_index(dir_files[0], 2)
file_harvester = AdditiveSequentialFileHarvester(
CONFIG, memento, self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 3)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 2)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_multi_file(self):
"""
Set the timing so the harvester finds multiple new files at once
"""
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# set the file filler to generate files with only 1.5 secs between,
# meaning 3 files will appear in the 5 seconds between the
# harvester checking
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 9, 1.5)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(
CONFIG, memento, self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# start a new event which will copy the first file and increase the
# file index into data directory with a delay in between
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 2)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()
def test_harvester_multi_file(self):
"""
Set the timing so the harvester finds multiple new files at once
"""
# start the harvester from scratch
memento = None
file_harvester = AdditiveSequentialFileHarvester(
CONFIG, memento, self.new_file_found_callback,
self.file_exception_callback)
file_harvester.start()
# set the file filler to generate files with only 1.5 secs between,
# meaning 3 files will appear in the 5 seconds between the
# harvester checking
self.directory_filler = gevent.spawn(self.fill_directory_with_files,
CONFIG['directory'],
CONFIG['pattern'], 9, 1.5)
# Wait for three sets of new files to be discovered
self.wait_for_file(0)
self.wait_for_file(self.found_file_count)
self.wait_for_file(self.found_file_count)
file_harvester.shutdown()