def test_file_mod_wait_time(self): """ Test that the harvester waits the required amount of time before finding files that are being modified """ # just touch the file to create it file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # keep track of how long it takes to find the file approximately file_found_time = 0; while(not self.data_found): time.sleep(1) file_found_time += 1 if file_found_time > 60: raise Exception("Timeout waiting to find file") if file_found_time < CONFIG.get(DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME): # we found the file before the mod time, this is bad! file_harvester.shutdown() self.fail('Files found in %s seconds' % file_found_time) log.debug('File found in %s seconds', file_found_time) file_harvester.shutdown()
def test_harvester_with_initial_data(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 1) # Wait for three lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) memento = self.get_file_metadata(FILENAME) log.debug('Starting with memento %s', memento) self.starting_lines = 2 # start the harvester at the end of the current data file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # just touch the file so it is created file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) memento = self.get_file_metadata(FILENAME) log.debug('Starting with memento %s', memento) self.starting_lines = 2 # start the harvester at the end of the current data file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0) # start the harvester at the end of the current data self.current_offset = 32 file_offset = 32 file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 2) # Wait for lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def test_harvester_with_initial_data(self): """ Test that the harvester can find the file changes from a file already in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() self.wait_for_data(2) # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_multi_line(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start the harvester from scratch file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 4, .9) # Wait for lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def test_file_mod_wait_time(self): """ Test that the harvester waits the required amount of time before finding files that are being modified """ # just touch the file to create it file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # keep track of how long it takes to find the file approximately file_found_time = 0 while (not self.data_found): time.sleep(1) file_found_time += 1 if file_found_time > 60: raise Exception("Timeout waiting to find file") if file_found_time < CONFIG.get( DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME): # we found the file before the mod time, this is bad! file_harvester.shutdown() self.fail('Files found in %s seconds' % file_found_time) log.debug('File found in %s seconds', file_found_time) file_harvester.shutdown()
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # just touch the file so it is created file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_initial_data(self): """ Test that the harvester can find the file changes from a file already in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() self.wait_for_data(2) # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()