def test_missing_directory(self): config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']} self.clean_directory(TESTDIR) os.rmdir(TESTDIR) self.assertFalse(os.path.exists(TESTDIR)) # start the harvester from scratch memento = None os.mkdir(TESTDIR) file_harvester = SortingDirectoryHarvester(config, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 2) # Wait for three sets of new files to be discovered self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_harvester_multi_file(self): """ Set the timing so the harvester finds multiple new files at once """ # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester(CONFIG, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # set the file filler to generate files with only .5 secs between, # meaning 2 files will appear in the 1 seconds between the # harvester checking self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 12, .5) # Wait for sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester(CONFIG, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 6) # Wait for three sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_harvester_with_memento(self): """ Test that the harvester can find file as they are added to a directory, using a memento to start partway through the indices """ # make sure we have 2 files already in the directory self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'], 0, 2, 0) # start at index 2 memento = CONFIG['directory'] + '/' + 'unit_' + INDICIES[1] + CONFIG['pattern'].replace('*', '') log.debug("starting with memento %s", memento) file_harvester = SortingDirectoryHarvester(CONFIG, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES # with a delay in between self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 2, 9) # Wait for three sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def _build_harvester(self, harvester_state): self._harvester = SortingDirectoryHarvester(self._harvester_config, harvester_state, self._new_file_callback, self._exception_callback) return self._harvester
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester( CONFIG, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 6) # Wait for three sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_harvester_multi_file(self): """ Set the timing so the harvester finds multiple new files at once """ # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester( CONFIG, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # set the file filler to generate files with only .5 secs between, # meaning 2 files will appear in the 1 seconds between the # harvester checking self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 12, .5) # Wait for sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_init(self): """ Test initialize """ config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']} # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester(config, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.sort_files(['a_1_2.bla', 'a_2_2.bla']) file_harvester.start() file_harvester.shutdown()
def test_harvester_without_frequency(self): """ Test that we can use a default frequency """ config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']} # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester(config, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 2) # Wait for three sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()
def test_harvester_without_frequency(self): """ Test that we can use a default frequency """ config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']} # start the harvester from scratch memento = None file_harvester = SortingDirectoryHarvester( config, memento, self.new_file_found_callback, self.file_exception_callback) file_harvester.start() # start a new event which will increase the file index using INDICIES self.directory_filler = gevent.spawn(self.fill_directory_with_files, CONFIG['directory'], CONFIG['pattern'], 0, 2) # Wait for three sets of new files to be discovered self.wait_for_file(0) self.wait_for_file(self.found_file_count) self.wait_for_file(self.found_file_count) file_harvester.shutdown()