def test_missing_directory(self):
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        self.clean_directory(TESTDIR)
        os.rmdir(TESTDIR)
        self.assertFalse(os.path.exists(TESTDIR))

        # start the harvester from scratch
        memento = None

        os.mkdir(TESTDIR)
        file_harvester = SortingDirectoryHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)
        file_harvester.start()

        # start a new event which will increase the file index using INDICIES
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
 def test_harvester_multi_file(self):
     """
     Set the timing so the harvester finds multiple new files at once
     """
     
     # start the harvester from scratch
     memento = None
     file_harvester = SortingDirectoryHarvester(CONFIG, memento,
                                                 self.new_file_found_callback,
                                                 self.file_exception_callback)
     file_harvester.start()
     
     # set the file filler to generate files with only .5 secs between,
     # meaning 2 files will appear in the 1 seconds between the
     # harvester checking
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 0, 12, .5)
     
     # Wait for sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     
     file_harvester.shutdown()
 def test_harvester_from_scratch(self):
     """
     Test that the harvester can find files as they are added to a directory,
     starting with just the base file in the directory
     """
     # start the harvester from scratch
     memento = None
     file_harvester = SortingDirectoryHarvester(CONFIG, memento,
                                                self.new_file_found_callback,
                                                self.file_exception_callback)
     file_harvester.start()
     
     # start a new event which will increase the file index using INDICIES
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 0, 6)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
 def test_harvester_with_memento(self):
     """
     Test that the harvester can find file as they are added to a directory,
     using a memento to start partway through the indices
     """
     
     # make sure we have 2 files already in the directory
     self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'], 0, 2, 0)
     
     # start at index 2
     memento = CONFIG['directory'] + '/' + 'unit_' + INDICIES[1] + CONFIG['pattern'].replace('*', '')
     log.debug("starting with memento %s", memento)
     file_harvester = SortingDirectoryHarvester(CONFIG, memento,
                                                self.new_file_found_callback,
                                                self.file_exception_callback)
     file_harvester.start()
     
     # start a new event which will increase the file index using INDICIES
     # with a delay in between
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 2, 9)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
Exemplo n.º 5
0
    def _build_harvester(self, harvester_state):
        self._harvester = SortingDirectoryHarvester(self._harvester_config,
                                                    harvester_state,
                                                    self._new_file_callback,
                                                    self._exception_callback)

        return self._harvester
    def test_harvester_from_scratch(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(
            CONFIG, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # start a new event which will increase the file index using INDICIES
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 6)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
    def test_harvester_multi_file(self):
        """
        Set the timing so the harvester finds multiple new files at once
        """

        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(
            CONFIG, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # set the file filler to generate files with only .5 secs between,
        # meaning 2 files will appear in the 1 seconds between the
        # harvester checking
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 12, .5)

        # Wait for sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
Exemplo n.º 8
0
    def test_missing_directory(self):
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        self.clean_directory(TESTDIR)
        os.rmdir(TESTDIR)
        self.assertFalse(os.path.exists(TESTDIR))

        # start the harvester from scratch
        memento = None

        os.mkdir(TESTDIR)
        file_harvester = SortingDirectoryHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)
        file_harvester.start()

        # start a new event which will increase the file index using INDICIES
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
Exemplo n.º 9
0
 def test_harvester_with_memento(self):
     """
     Test that the harvester can find file as they are added to a directory,
     using a memento to start partway through the indices
     """
     
     # make sure we have 2 files already in the directory
     self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'], 0, 2, 0)
     
     # start at index 2
     memento = CONFIG['directory'] + '/' + 'unit_' + INDICIES[1] + CONFIG['pattern'].replace('*', '')
     log.debug("starting with memento %s", memento)
     file_harvester = SortingDirectoryHarvester(CONFIG, memento,
                                                self.new_file_found_callback,
                                                self.file_exception_callback)
     file_harvester.start()
     
     # start a new event which will increase the file index using INDICIES
     # with a delay in between
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 2, 9)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
Exemplo n.º 10
0
    def test_init(self):
        """
        Test initialize
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)

        file_harvester.sort_files(['a_1_2.bla', 'a_2_2.bla'])
        file_harvester.start()
        file_harvester.shutdown()
    def test_harvester_without_frequency(self):
        """
        Test that we can use a default frequency
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)
        file_harvester.start()

        # start a new event which will increase the file index using INDICIES
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
    def test_init(self):
        """
        Test initialize
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)

        file_harvester.sort_files(['a_1_2.bla', 'a_2_2.bla'])
        file_harvester.start()
        file_harvester.shutdown()
    def test_harvester_without_frequency(self):
        """
        Test that we can use a default frequency
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = SortingDirectoryHarvester(
            config, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # start a new event which will increase the file index using INDICIES
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 0, 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()