def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) memento = self.get_file_metadata(FILENAME) log.debug('Starting with memento %s', memento) self.starting_lines = 2 # start the harvester at the end of the current data file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_initial_data(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 1) # Wait for three lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def _build_harvester(self, driver_state): """ Build the harvester @param driver_state The starting driver state """ self._harvester = [] if DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED in self._harvester_config: telemetered_harvester = SingleFileHarvester( self._harvester_config.get( DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED), driver_state[DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED], lambda file_state: self._file_changed_callback( file_state, DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED), self._exception_callback) self._harvester.append(telemetered_harvester) else: log.warn( 'No configuration for telemetered harvester, not building') if DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED in self._harvester_config: recovered_harvester = SingleDirectoryHarvester( self._harvester_config.get( DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), driver_state[DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED], lambda filename: self._new_file_callback( filename, DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), lambda modified: self._modified_file_callback( modified, DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED), self._exception_callback) self._harvester.append(recovered_harvester) else: log.warn('No configuration for recovered harvester, not building') return self._harvester
def _build_harvester(self, driver_state): """ Build the harvester @param driver_state The starting driver state """ harvesters = [] if DataSourceKey.FLORT_DJ_SIO_TELEMETERED in self._harvester_config: telem_harvester = SingleFileHarvester( self._harvester_config.get( DataSourceKey.FLORT_DJ_SIO_TELEMETERED), driver_state[DataSourceKey.FLORT_DJ_SIO_TELEMETERED], lambda file_state: self._file_changed_callback( file_state, DataSourceKey.FLORT_DJ_SIO_TELEMETERED), self._exception_callback) harvesters.append(telem_harvester) else: log.warn('No configuration for %s harvester, not building', DataSourceKey.FLORT_DJ_SIO_TELEMETERED) if DataSourceKey.FLORT_DJ_SIO_RECOVERED in self._harvester_config: recov_harvester = SingleDirectoryHarvester( self._harvester_config.get( DataSourceKey.FLORT_DJ_SIO_RECOVERED), driver_state[DataSourceKey.FLORT_DJ_SIO_RECOVERED], lambda filename: self._new_file_callback( filename, DataSourceKey.FLORT_DJ_SIO_RECOVERED), lambda modified: self._modified_file_callback( modified, DataSourceKey.FLORT_DJ_SIO_RECOVERED), self._exception_callback) harvesters.append(recov_harvester) else: log.warn('No configuration for %s harvester, not building', DataSourceKey.FLORT_DJ_SIO_RECOVERED) return harvesters
def test_file_mod_wait_time(self): """ Test that the harvester waits the required amount of time before finding files that are being modified """ # just touch the file to create it file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # keep track of how long it takes to find the file approximately file_found_time = 0; while(not self.data_found): time.sleep(1) file_found_time += 1 if file_found_time > 60: raise Exception("Timeout waiting to find file") if file_found_time < CONFIG.get(DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME): # we found the file before the mod time, this is bad! file_harvester.shutdown() self.fail('Files found in %s seconds' % file_found_time) log.debug('File found in %s seconds', file_found_time) file_harvester.shutdown()
def _build_harvester(self, driver_state): """ Build and return the harvester """ self._harvester = SingleFileHarvester(self._harvester_config, driver_state, self._file_changed_callback, self._exception_callback) return self._harvester
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # just touch the file so it is created file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) memento = self.get_file_metadata(FILENAME) log.debug('Starting with memento %s', memento) self.starting_lines = 2 # start the harvester at the end of the current data file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_with_offset(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0) # start the harvester at the end of the current data self.current_offset = 32 file_offset = 32 file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 2) # Wait for lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def test_harvester_multi_line(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # start the harvester from scratch file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG['directory'], CONFIG['filename'], 4, .9) # Wait for lines of new data to be discovered self.wait_for_data() self.wait_for_data() file_harvester.shutdown()
def test_harvester_with_initial_data(self): """ Test that the harvester can find the file changes from a file already in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() self.wait_for_data(2) # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def test_harvester_from_scratch(self): """ Test that the harvester can find files as they are added to a directory, starting with just the base file in the directory """ # just touch the file so it is created file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn(self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def _build_harvester(self, driver_state): """ Build and return the harvesters """ harvesters = [] # list of harvesters to be returned # # Verify that the WFP harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.VEL3D_L_WFP in self._harvester_config: wfp_harvester = SingleDirectoryHarvester( self._harvester_config.get(DataTypeKey.VEL3D_L_WFP), driver_state[DataTypeKey.VEL3D_L_WFP], lambda filename: self._new_file_callback( filename, DataTypeKey.VEL3D_L_WFP), lambda modified: self._modified_file_callback( modified, DataTypeKey.VEL3D_L_WFP), self._exception_callback) if wfp_harvester is not None: harvesters.append(wfp_harvester) else: log.warn('Missing harvester configuration for key %s', DataTypeKey.VEL3D_L_WFP) # # Verify that the SIO Mule harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.VEL3D_L_WFP_SIO_MULE in self._harvester_config: sio_harvester = SingleFileHarvester( self._harvester_config.get(DataTypeKey.VEL3D_L_WFP_SIO_MULE), driver_state[DataTypeKey.VEL3D_L_WFP_SIO_MULE], lambda file_state: self._file_changed_callback( file_state, DataTypeKey.VEL3D_L_WFP_SIO_MULE), self._exception_callback) if sio_harvester is not None: harvesters.append(sio_harvester) else: log.warn('Missing harvester configuration for key %s', DataTypeKey.VEL3D_L_WFP) return harvesters
def _build_harvester(self, driver_state): """ Build and return the harvester """ self._harvester = [] if DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED in self._harvester_config: telemetered_harvester = SingleFileHarvester( self._harvester_config.get( DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED), driver_state[DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED], lambda file_state: self._file_changed_callback( file_state, DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED), self._exception_callback) self._harvester.append(telemetered_harvester) else: log.warn( 'No configuration for telemetered harvester, not building') return self._harvester
def _build_harvester(self, driver_state): """ Build and return the harvesters """ harvesters = [] # list of harvesters to be returned # # Verify that the WFP_ENG_STC_IMODEM harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.WFP_ENG_STC_IMODEM in self._harvester_config: wfp_harvester = SingleDirectoryHarvester( self._harvester_config.get(DataTypeKey.WFP_ENG_STC_IMODEM), driver_state[DataTypeKey.WFP_ENG_STC_IMODEM], lambda filename: self._new_file_callback( filename, DataTypeKey.WFP_ENG_STC_IMODEM), lambda modified: self._modified_file_callback( modified, DataTypeKey.WFP_ENG_STC_IMODEM), self._exception_callback) if wfp_harvester is not None: harvesters.append(wfp_harvester) else: log.debug('WFP_ENG_STC_IMODEM HARVESTER NOT BUILT') # # Verify that the WFP_ENG_WFP_SIO_MULE harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.WFP_ENG_WFP_SIO_MULE in self._harvester_config: sio_harvester = SingleFileHarvester( self._harvester_config.get(DataTypeKey.WFP_ENG_WFP_SIO_MULE), driver_state[DataTypeKey.WFP_ENG_WFP_SIO_MULE], lambda file_state: self._file_changed_callback( file_state, DataTypeKey.WFP_ENG_WFP_SIO_MULE), self._exception_callback) if sio_harvester is not None: harvesters.append(sio_harvester) else: log.debug('WFP_ENG_WFP_SIO_MULE HARVESTER NOT BUILT') return harvesters
def test_file_mod_wait_time(self): """ Test that the harvester waits the required amount of time before finding files that are being modified """ # just touch the file to create it file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME) open(file_path, 'a').close() # start the harvester from scratch memento = None file_harvester = SingleFileHarvester(CONFIG, memento, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() # set the filler to write 1 line just under every second, meaning that 2 lines will # appear for each wait_for_data self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # keep track of how long it takes to find the file approximately file_found_time = 0 while (not self.data_found): time.sleep(1) file_found_time += 1 if file_found_time > 60: raise Exception("Timeout waiting to find file") if file_found_time < CONFIG.get( DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME): # we found the file before the mod time, this is bad! file_harvester.shutdown() self.fail('Files found in %s seconds' % file_found_time) log.debug('File found in %s seconds', file_found_time) file_harvester.shutdown()
def _build_harvester(self, driver_state): """ Build the telemetered and recovered harvester if they are configured @param driver_state The starting driver state """ harvesters = [] if DataSourceKey.DOSTA_LN_WFP_SIO_MULE in self._harvester_config: telem_harvester = SingleFileHarvester( self._harvester_config.get( DataSourceKey.DOSTA_LN_WFP_SIO_MULE), driver_state[DataSourceKey.DOSTA_LN_WFP_SIO_MULE], lambda file_state: self._file_changed_callback( file_state, DataSourceKey.DOSTA_LN_WFP_SIO_MULE), self._exception_callback) harvesters.append(telem_harvester) else: log.warn( 'No configuration for dosta ln wfp sio mule harvester, not building' ) if DataSourceKey.DOSTA_LN_WFP in self._harvester_config: recov_harvester = SingleDirectoryHarvester( self._harvester_config.get(DataSourceKey.DOSTA_LN_WFP), driver_state[DataSourceKey.DOSTA_LN_WFP], lambda filename: self._new_file_callback( filename, DataSourceKey.DOSTA_LN_WFP), lambda modified: self._modified_file_callback( modified, DataSourceKey.DOSTA_LN_WFP), self._exception_callback) harvesters.append(recov_harvester) else: log.warn( 'No configuration for dosta ln wfp harvester, not building') return harvesters
def test_harvester_with_initial_data(self): """ Test that the harvester can find the file changes from a file already in the directory """ # start with 2 lines in the file self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0) # start the harvester with data in the file file_offset = None file_harvester = SingleFileHarvester(CONFIG, file_offset, self.new_data_found_callback, self.harvester_exception_callback) file_harvester.start() self.wait_for_data(2) # start a new event which will write new lines of data into the file self.file_filler = gevent.spawn( self.fill_file_with_data, CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2) # Wait for two lines of new data to be discovered self.wait_for_data(2) file_harvester.shutdown()
def _build_harvester(self, driver_state): """ Build the harvester @param driver_state The starting driver state """ harvesters = [] # # Verify that the CO Recovered harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.CTDMO_GHQR_CO in self._harvester_config: co_harvester = SingleDirectoryHarvester( self._harvester_config.get(DataTypeKey.CTDMO_GHQR_CO), driver_state[DataTypeKey.CTDMO_GHQR_CO], lambda filename: self._new_file_callback( filename, DataTypeKey.CTDMO_GHQR_CO), lambda modified: self._modified_file_callback( modified, DataTypeKey.CTDMO_GHQR_CO), self._exception_callback) if co_harvester is not None: harvesters.append(co_harvester) else: log.warn('Could not build ctdmo_ghqr_co harvester') else: log.warn( 'No configuration for ctdmo_ghqr_co harvester, not building') # # Verify that the CT Recovered harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.CTDMO_GHQR_CT in self._harvester_config: ct_harvester = SingleDirectoryHarvester( self._harvester_config.get(DataTypeKey.CTDMO_GHQR_CT), driver_state[DataTypeKey.CTDMO_GHQR_CT], lambda filename: self._new_file_callback( filename, DataTypeKey.CTDMO_GHQR_CT), lambda modified: self._modified_file_callback( modified, DataTypeKey.CTDMO_GHQR_CT), self._exception_callback) if ct_harvester is not None: harvesters.append(ct_harvester) else: log.warn('Could not build ctdmo_ghqr_ct harvester') else: log.warn( 'No configuration for ctdmo_ghqr_ct harvester, not building') # # Verify that the CT Recovered harvester has been configured. # If so, build the harvester and add it to the list of harvesters. # if DataTypeKey.CTDMO_GHQR_SIO_MULE in self._harvester_config: ctdmo_ghqr_sio_mule_harvester = SingleFileHarvester( self._harvester_config.get(DataTypeKey.CTDMO_GHQR_SIO_MULE), driver_state[DataTypeKey.CTDMO_GHQR_SIO_MULE], lambda file_state: self._file_changed_callback( file_state, DataTypeKey.CTDMO_GHQR_SIO_MULE), self._exception_callback) if ctdmo_ghqr_sio_mule_harvester is not None: harvesters.append(ctdmo_ghqr_sio_mule_harvester) else: log.warn('Could not build ctdmo_ghqr_sio_mule harvester') else: log.warn( 'No configuration for ctdmo_ghqr_sio_mule harvester, not building' ) return harvesters