def test_file_mod_wait_time(self):
        """
        Test that the harvester waits the required amount of time before finding
        files that are being modified
        """
        # just touch the file to create it
        file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
        open(file_path, 'a').close()

        # start the harvester from scratch
        memento = None
        file_harvester = SingleFileHarvester(CONFIG, memento, 
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # set the filler to write 1 line just under every second, meaning that 2 lines will
        # appear for each wait_for_data
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                             CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                             FILENAME, 2, 0)
        # keep track of how long it takes to find the file approximately
	file_found_time = 0;
        while(not self.data_found):
            time.sleep(1)
	    file_found_time += 1
            if file_found_time > 60:
                raise Exception("Timeout waiting to find file")

	if file_found_time < CONFIG.get(DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
	    # we found the file before the mod time, this is bad!
	    file_harvester.shutdown()
	    self.fail('Files found in %s seconds' % file_found_time)
	log.debug('File found in %s seconds', file_found_time)
	file_harvester.shutdown()
 def test_harvester_with_initial_data(self):
     """
     Test that the harvester can find files as they are added to a directory,
     starting with just the base file in the directory
     """
     # start with 2 lines in the file
     self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)
     
     # start the harvester with data in the file
     file_offset = None
     file_harvester = SingleFileHarvester(CONFIG, file_offset,
                                          self.new_data_found_callback,
                                          self.harvester_exception_callback)
     file_harvester.start()
     
     # start a new event which will write new lines of data into the file
     self.file_filler = gevent.spawn(self.fill_file_with_data,
                                          CONFIG['directory'],
                                          CONFIG['filename'], 1)
     
     # Wait for three lines of new data to be discovered
     self.wait_for_data()
     self.wait_for_data()
      
     file_harvester.shutdown()
    def test_harvester_with_offset(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start with 2 lines in the file
        self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
        memento = self.get_file_metadata(FILENAME)
        log.debug('Starting with memento %s', memento)
        self.starting_lines = 2

        # start the harvester at the end of the current data
        file_harvester = SingleFileHarvester(CONFIG, memento, 
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                        CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                        FILENAME, 2)

        # Wait for lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()
Exemple #4
0
    def test_harvester_from_scratch(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # just touch the file so it is created
        file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
                                 FILENAME)
        open(file_path, 'a').close()

        # start the harvester from scratch
        memento = None
        file_harvester = SingleFileHarvester(CONFIG, memento,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(
            self.fill_file_with_data,
            CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)

        # Wait for two lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()
Exemple #5
0
    def test_harvester_with_offset(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start with 2 lines in the file
        self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                 FILENAME, 2, 0)
        memento = self.get_file_metadata(FILENAME)
        log.debug('Starting with memento %s', memento)
        self.starting_lines = 2

        # start the harvester at the end of the current data
        file_harvester = SingleFileHarvester(CONFIG, memento,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(
            self.fill_file_with_data,
            CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)

        # Wait for lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()
    def test_harvester_with_offset(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start with 2 lines in the file
        self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)

        # start the harvester at the end of the current data
        self.current_offset = 32
        file_offset = 32
        file_harvester = SingleFileHarvester(CONFIG, file_offset,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                        CONFIG['directory'],
                                        CONFIG['filename'], 2)

        # Wait for lines of new data to be discovered
        self.wait_for_data()
        self.wait_for_data()

        file_harvester.shutdown()
    def test_harvester_with_initial_data(self):
        """
        Test that the harvester can find the file changes from a file already in the directory
        """
        # start with 2 lines in the file
        self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)

        # start the harvester with data in the file
        file_offset = None
        file_harvester = SingleFileHarvester(CONFIG, file_offset, 
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()
        self.wait_for_data(2)

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                             CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                             FILENAME, 2)

        # Wait for two lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()
 def test_harvester_multi_line(self):
     """
     Test that the harvester can find files as they are added to a directory,
     starting with just the base file in the directory
     """
     # start the harvester from scratch
     file_offset = None
     file_harvester = SingleFileHarvester(CONFIG, file_offset,
                                          self.new_data_found_callback,
                                          self.harvester_exception_callback)
     file_harvester.start()
     
     # set the filler to write 1 line just under every second, meaning that 2 lines will
     # appear for each wait_for_data
     self.file_filler = gevent.spawn(self.fill_file_with_data,
                                          CONFIG['directory'],
                                          CONFIG['filename'], 4, .9)
     
     # Wait for lines of new data to be discovered
     self.wait_for_data()
     self.wait_for_data()
      
     file_harvester.shutdown()
    def test_harvester_multi_line(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # start the harvester from scratch
        file_offset = None
        file_harvester = SingleFileHarvester(CONFIG, file_offset,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # set the filler to write 1 line just under every second, meaning that 2 lines will
        # appear for each wait_for_data
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                        CONFIG['directory'],
                                        CONFIG['filename'], 4, .9)

        # Wait for lines of new data to be discovered
        self.wait_for_data()
        self.wait_for_data()

        file_harvester.shutdown()
Exemple #10
0
    def test_file_mod_wait_time(self):
        """
        Test that the harvester waits the required amount of time before finding
        files that are being modified
        """
        # just touch the file to create it
        file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
                                 FILENAME)
        open(file_path, 'a').close()

        # start the harvester from scratch
        memento = None
        file_harvester = SingleFileHarvester(CONFIG, memento,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # set the filler to write 1 line just under every second, meaning that 2 lines will
        # appear for each wait_for_data
        self.file_filler = gevent.spawn(
            self.fill_file_with_data,
            CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
        # keep track of how long it takes to find the file approximately
        file_found_time = 0
        while (not self.data_found):
            time.sleep(1)
            file_found_time += 1
            if file_found_time > 60:
                raise Exception("Timeout waiting to find file")

        if file_found_time < CONFIG.get(
                DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
            # we found the file before the mod time, this is bad!
            file_harvester.shutdown()
            self.fail('Files found in %s seconds' % file_found_time)
        log.debug('File found in %s seconds', file_found_time)
        file_harvester.shutdown()
    def test_harvester_from_scratch(self):
        """
        Test that the harvester can find files as they are added to a directory,
        starting with just the base file in the directory
        """
        # just touch the file so it is created
	file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
        open(file_path, 'a').close()

        # start the harvester from scratch
        memento = None
        file_harvester = SingleFileHarvester(CONFIG, memento, 
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(self.fill_file_with_data,
                                             CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                             FILENAME, 2)

        # Wait for two lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()
Exemple #12
0
    def test_harvester_with_initial_data(self):
        """
        Test that the harvester can find the file changes from a file already in the directory
        """
        # start with 2 lines in the file
        self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
                                 FILENAME, 2, 0)

        # start the harvester with data in the file
        file_offset = None
        file_harvester = SingleFileHarvester(CONFIG, file_offset,
                                             self.new_data_found_callback,
                                             self.harvester_exception_callback)
        file_harvester.start()
        self.wait_for_data(2)

        # start a new event which will write new lines of data into the file
        self.file_filler = gevent.spawn(
            self.fill_file_with_data,
            CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)

        # Wait for two lines of new data to be discovered
        self.wait_for_data(2)
        file_harvester.shutdown()