def test_harvester_without_frequency(self):
        """
        Test that we can use a default frequency
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = AdditiveSequentialFileHarvester(config, memento,
                                                         self.new_file_found_callback,
                                                         self.file_exception_callback)
        file_harvester.start()

        # start a new event which will copy the first file and increase the
        # file index into data directory with a delay in between
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
 def test_harvester_with_memento(self):
     """
     Test that the harvester can find file as they are added to a directory,
     using a memento to start partway through the indices
     """
     # make sure we have 2 files already in the directory
     self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'], 2, 0)
     
     # start at index 2
     dir_files = glob.glob(CONFIG['directory'] + '/' + CONFIG['pattern'])
     memento = self.replace_file_index(dir_files[0], 2)
     file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
                                                      self.new_file_found_callback,
                                                      self.file_exception_callback)
     file_harvester.start()
     
     # start a new event which will copy the first file and increase the
     # file index into data directory with a delay in between
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 3)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
예제 #3
0
    def _build_harvester(self, harvester_state):
        self._harvester = AdditiveSequentialFileHarvester(
            self._harvester_config,
            harvester_state,
            self._new_file_callback,
            self._exception_callback
        )

        return self._harvester
    def test_harvester_without_frequency(self):
        """
        Test that we can use a default frequency
        """
        config = {'directory': TESTDIR, 'pattern': CONFIG['pattern']}

        # start the harvester from scratch
        memento = None
        file_harvester = AdditiveSequentialFileHarvester(
            config, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # start a new event which will copy the first file and increase the
        # file index into data directory with a delay in between
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
    def test_harvester_with_memento(self):
        """
        Test that the harvester can find file as they are added to a directory,
        using a memento to start partway through the indices
        """
        # make sure we have 2 files already in the directory
        self.fill_directory_with_files(CONFIG['directory'], CONFIG['pattern'],
                                       2, 0)

        # start at index 2
        dir_files = glob.glob(CONFIG['directory'] + '/' + CONFIG['pattern'])
        memento = self.replace_file_index(dir_files[0], 2)
        file_harvester = AdditiveSequentialFileHarvester(
            CONFIG, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # start a new event which will copy the first file and increase the
        # file index into data directory with a delay in between
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 3)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
 def test_harvester_from_scratch(self):
     """
     Test that the harvester can find files as they are added to a directory,
     starting with just the base file in the directory
     """
     # start the harvester from scratch
     memento = None
     file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
                                                      self.new_file_found_callback,
                                                      self.file_exception_callback)
     file_harvester.start()
     
     # start a new event which will copy the first file and increase the
     # file index into data directory with a delay in between
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 2)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
 def test_harvester_multi_file(self):
     """
     Set the timing so the harvester finds multiple new files at once
     """
     # start the harvester from scratch
     memento = None
     file_harvester = AdditiveSequentialFileHarvester(CONFIG, memento,
                                                      self.new_file_found_callback,
                                                      self.file_exception_callback)
     file_harvester.start()
     
     # set the file filler to generate files with only 1.5 secs between,
     # meaning 3 files will appear in the 5 seconds between the
     # harvester checking
     self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                          CONFIG['directory'],
                                          CONFIG['pattern'], 9, 1.5)
     
     # Wait for three sets of new files to be discovered
     self.wait_for_file(0)
     self.wait_for_file(self.found_file_count)
     self.wait_for_file(self.found_file_count)
      
     file_harvester.shutdown()
    def test_harvester_from_scratch(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start the harvester from scratch
        memento = None
        file_harvester = AdditiveSequentialFileHarvester(
            CONFIG, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # start a new event which will copy the first file and increase the
        # file index into data directory with a delay in between
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 2)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()
    def test_harvester_multi_file(self):
        """
        Set the timing so the harvester finds multiple new files at once
        """
        # start the harvester from scratch
        memento = None
        file_harvester = AdditiveSequentialFileHarvester(
            CONFIG, memento, self.new_file_found_callback,
            self.file_exception_callback)
        file_harvester.start()

        # set the file filler to generate files with only 1.5 secs between,
        # meaning 3 files will appear in the 5 seconds between the
        # harvester checking
        self.directory_filler = gevent.spawn(self.fill_directory_with_files,
                                             CONFIG['directory'],
                                             CONFIG['pattern'], 9, 1.5)

        # Wait for three sets of new files to be discovered
        self.wait_for_file(0)
        self.wait_for_file(self.found_file_count)
        self.wait_for_file(self.found_file_count)

        file_harvester.shutdown()