def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
memento = self.get_file_metadata(FILENAME)
log.debug('Starting with memento %s', memento)
self.starting_lines = 2
# start the harvester at the end of the current data
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 1)
# Wait for three lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def _build_harvester(self, driver_state):
"""
Build the harvester
@param driver_state The starting driver state
"""
self._harvester = []
if DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED in self._harvester_config:
telemetered_harvester = SingleFileHarvester(
self._harvester_config.get(
DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED),
driver_state[DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED],
lambda file_state: self._file_changed_callback(
file_state, DataTypeKey.DOSTA_ABCDJM_SIO_TELEMETERED),
self._exception_callback)
self._harvester.append(telemetered_harvester)
else:
log.warn(
'No configuration for telemetered harvester, not building')
if DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED in self._harvester_config:
recovered_harvester = SingleDirectoryHarvester(
self._harvester_config.get(
DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED),
driver_state[DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED],
lambda filename: self._new_file_callback(
filename, DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED),
lambda modified: self._modified_file_callback(
modified, DataTypeKey.DOSTA_ABCDJM_SIO_RECOVERED),
self._exception_callback)
self._harvester.append(recovered_harvester)
else:
log.warn('No configuration for recovered harvester, not building')
return self._harvester
def _build_harvester(self, driver_state):
"""
Build the harvester
@param driver_state The starting driver state
"""
harvesters = []
if DataSourceKey.FLORT_DJ_SIO_TELEMETERED in self._harvester_config:
telem_harvester = SingleFileHarvester(
self._harvester_config.get(
DataSourceKey.FLORT_DJ_SIO_TELEMETERED),
driver_state[DataSourceKey.FLORT_DJ_SIO_TELEMETERED],
lambda file_state: self._file_changed_callback(
file_state, DataSourceKey.FLORT_DJ_SIO_TELEMETERED),
self._exception_callback)
harvesters.append(telem_harvester)
else:
log.warn('No configuration for %s harvester, not building',
DataSourceKey.FLORT_DJ_SIO_TELEMETERED)
if DataSourceKey.FLORT_DJ_SIO_RECOVERED in self._harvester_config:
recov_harvester = SingleDirectoryHarvester(
self._harvester_config.get(
DataSourceKey.FLORT_DJ_SIO_RECOVERED),
driver_state[DataSourceKey.FLORT_DJ_SIO_RECOVERED],
lambda filename: self._new_file_callback(
filename, DataSourceKey.FLORT_DJ_SIO_RECOVERED),
lambda modified: self._modified_file_callback(
modified, DataSourceKey.FLORT_DJ_SIO_RECOVERED),
self._exception_callback)
harvesters.append(recov_harvester)
else:
log.warn('No configuration for %s harvester, not building',
DataSourceKey.FLORT_DJ_SIO_RECOVERED)
return harvesters
def test_file_mod_wait_time(self):
"""
Test that the harvester waits the required amount of time before finding
files that are being modified
"""
# just touch the file to create it
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
# keep track of how long it takes to find the file approximately
file_found_time = 0;
while(not self.data_found):
time.sleep(1)
file_found_time += 1
if file_found_time > 60:
raise Exception("Timeout waiting to find file")
if file_found_time < CONFIG.get(DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
# we found the file before the mod time, this is bad!
file_harvester.shutdown()
self.fail('Files found in %s seconds' % file_found_time)
log.debug('File found in %s seconds', file_found_time)
file_harvester.shutdown()
def _build_harvester(self, driver_state):
"""
Build and return the harvester
"""
self._harvester = SingleFileHarvester(self._harvester_config,
driver_state,
self._file_changed_callback,
self._exception_callback)
return self._harvester
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# just touch the file so it is created
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
memento = self.get_file_metadata(FILENAME)
log.debug('Starting with memento %s', memento)
self.starting_lines = 2
# start the harvester at the end of the current data
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_with_offset(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG['directory'], CONFIG['filename'], 2, 0)
# start the harvester at the end of the current data
self.current_offset = 32
file_offset = 32
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 2)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_multi_line(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 4, .9)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find the file changes from a file already in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
self.wait_for_data(2)
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def test_harvester_from_scratch(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# just touch the file so it is created
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY), FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def _build_harvester(self, driver_state):
"""
Build and return the harvesters
"""
harvesters = [] # list of harvesters to be returned
#
# Verify that the WFP harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.VEL3D_L_WFP in self._harvester_config:
wfp_harvester = SingleDirectoryHarvester(
self._harvester_config.get(DataTypeKey.VEL3D_L_WFP),
driver_state[DataTypeKey.VEL3D_L_WFP],
lambda filename: self._new_file_callback(
filename, DataTypeKey.VEL3D_L_WFP),
lambda modified: self._modified_file_callback(
modified, DataTypeKey.VEL3D_L_WFP),
self._exception_callback)
if wfp_harvester is not None:
harvesters.append(wfp_harvester)
else:
log.warn('Missing harvester configuration for key %s',
DataTypeKey.VEL3D_L_WFP)
#
# Verify that the SIO Mule harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.VEL3D_L_WFP_SIO_MULE in self._harvester_config:
sio_harvester = SingleFileHarvester(
self._harvester_config.get(DataTypeKey.VEL3D_L_WFP_SIO_MULE),
driver_state[DataTypeKey.VEL3D_L_WFP_SIO_MULE],
lambda file_state: self._file_changed_callback(
file_state, DataTypeKey.VEL3D_L_WFP_SIO_MULE),
self._exception_callback)
if sio_harvester is not None:
harvesters.append(sio_harvester)
else:
log.warn('Missing harvester configuration for key %s',
DataTypeKey.VEL3D_L_WFP)
return harvesters
def _build_harvester(self, driver_state):
"""
Build and return the harvester
"""
self._harvester = []
if DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED in self._harvester_config:
telemetered_harvester = SingleFileHarvester(
self._harvester_config.get(
DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED),
driver_state[DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED],
lambda file_state: self._file_changed_callback(
file_state, DataSourceKey.SIO_ENG_SIO_MULE_TELEMETERED),
self._exception_callback)
self._harvester.append(telemetered_harvester)
else:
log.warn(
'No configuration for telemetered harvester, not building')
return self._harvester
def _build_harvester(self, driver_state):
"""
Build and return the harvesters
"""
harvesters = [] # list of harvesters to be returned
#
# Verify that the WFP_ENG_STC_IMODEM harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.WFP_ENG_STC_IMODEM in self._harvester_config:
wfp_harvester = SingleDirectoryHarvester(
self._harvester_config.get(DataTypeKey.WFP_ENG_STC_IMODEM),
driver_state[DataTypeKey.WFP_ENG_STC_IMODEM],
lambda filename: self._new_file_callback(
filename, DataTypeKey.WFP_ENG_STC_IMODEM),
lambda modified: self._modified_file_callback(
modified, DataTypeKey.WFP_ENG_STC_IMODEM),
self._exception_callback)
if wfp_harvester is not None:
harvesters.append(wfp_harvester)
else:
log.debug('WFP_ENG_STC_IMODEM HARVESTER NOT BUILT')
#
# Verify that the WFP_ENG_WFP_SIO_MULE harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.WFP_ENG_WFP_SIO_MULE in self._harvester_config:
sio_harvester = SingleFileHarvester(
self._harvester_config.get(DataTypeKey.WFP_ENG_WFP_SIO_MULE),
driver_state[DataTypeKey.WFP_ENG_WFP_SIO_MULE],
lambda file_state: self._file_changed_callback(
file_state, DataTypeKey.WFP_ENG_WFP_SIO_MULE),
self._exception_callback)
if sio_harvester is not None:
harvesters.append(sio_harvester)
else:
log.debug('WFP_ENG_WFP_SIO_MULE HARVESTER NOT BUILT')
return harvesters
def test_file_mod_wait_time(self):
"""
Test that the harvester waits the required amount of time before finding
files that are being modified
"""
# just touch the file to create it
file_path = os.path.join(CONFIG.get(DataSetDriverConfigKeys.DIRECTORY),
FILENAME)
open(file_path, 'a').close()
# start the harvester from scratch
memento = None
file_harvester = SingleFileHarvester(CONFIG, memento,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2, 0)
# keep track of how long it takes to find the file approximately
file_found_time = 0
while (not self.data_found):
time.sleep(1)
file_found_time += 1
if file_found_time > 60:
raise Exception("Timeout waiting to find file")
if file_found_time < CONFIG.get(
DataSetDriverConfigKeys.FILE_MOD_WAIT_TIME):
# we found the file before the mod time, this is bad!
file_harvester.shutdown()
self.fail('Files found in %s seconds' % file_found_time)
log.debug('File found in %s seconds', file_found_time)
file_harvester.shutdown()
def _build_harvester(self, driver_state):
"""
Build the telemetered and recovered harvester if they are configured
@param driver_state The starting driver state
"""
harvesters = []
if DataSourceKey.DOSTA_LN_WFP_SIO_MULE in self._harvester_config:
telem_harvester = SingleFileHarvester(
self._harvester_config.get(
DataSourceKey.DOSTA_LN_WFP_SIO_MULE),
driver_state[DataSourceKey.DOSTA_LN_WFP_SIO_MULE],
lambda file_state: self._file_changed_callback(
file_state, DataSourceKey.DOSTA_LN_WFP_SIO_MULE),
self._exception_callback)
harvesters.append(telem_harvester)
else:
log.warn(
'No configuration for dosta ln wfp sio mule harvester, not building'
)
if DataSourceKey.DOSTA_LN_WFP in self._harvester_config:
recov_harvester = SingleDirectoryHarvester(
self._harvester_config.get(DataSourceKey.DOSTA_LN_WFP),
driver_state[DataSourceKey.DOSTA_LN_WFP],
lambda filename: self._new_file_callback(
filename, DataSourceKey.DOSTA_LN_WFP),
lambda modified: self._modified_file_callback(
modified, DataSourceKey.DOSTA_LN_WFP),
self._exception_callback)
harvesters.append(recov_harvester)
else:
log.warn(
'No configuration for dosta ln wfp harvester, not building')
return harvesters
def test_harvester_multi_line(self):
"""
Test that the harvester can find files as they are added to a directory,
starting with just the base file in the directory
"""
# start the harvester from scratch
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
# set the filler to write 1 line just under every second, meaning that 2 lines will
# appear for each wait_for_data
self.file_filler = gevent.spawn(self.fill_file_with_data,
CONFIG['directory'],
CONFIG['filename'], 4, .9)
# Wait for lines of new data to be discovered
self.wait_for_data()
self.wait_for_data()
file_harvester.shutdown()
def test_harvester_with_initial_data(self):
"""
Test that the harvester can find the file changes from a file already in the directory
"""
# start with 2 lines in the file
self.fill_file_with_data(CONFIG[DataSetDriverConfigKeys.DIRECTORY],
FILENAME, 2, 0)
# start the harvester with data in the file
file_offset = None
file_harvester = SingleFileHarvester(CONFIG, file_offset,
self.new_data_found_callback,
self.harvester_exception_callback)
file_harvester.start()
self.wait_for_data(2)
# start a new event which will write new lines of data into the file
self.file_filler = gevent.spawn(
self.fill_file_with_data,
CONFIG[DataSetDriverConfigKeys.DIRECTORY], FILENAME, 2)
# Wait for two lines of new data to be discovered
self.wait_for_data(2)
file_harvester.shutdown()
def _build_harvester(self, driver_state):
"""
Build the harvester
@param driver_state The starting driver state
"""
harvesters = []
#
# Verify that the CO Recovered harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.CTDMO_GHQR_CO in self._harvester_config:
co_harvester = SingleDirectoryHarvester(
self._harvester_config.get(DataTypeKey.CTDMO_GHQR_CO),
driver_state[DataTypeKey.CTDMO_GHQR_CO],
lambda filename: self._new_file_callback(
filename, DataTypeKey.CTDMO_GHQR_CO),
lambda modified: self._modified_file_callback(
modified, DataTypeKey.CTDMO_GHQR_CO),
self._exception_callback)
if co_harvester is not None:
harvesters.append(co_harvester)
else:
log.warn('Could not build ctdmo_ghqr_co harvester')
else:
log.warn(
'No configuration for ctdmo_ghqr_co harvester, not building')
#
# Verify that the CT Recovered harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.CTDMO_GHQR_CT in self._harvester_config:
ct_harvester = SingleDirectoryHarvester(
self._harvester_config.get(DataTypeKey.CTDMO_GHQR_CT),
driver_state[DataTypeKey.CTDMO_GHQR_CT],
lambda filename: self._new_file_callback(
filename, DataTypeKey.CTDMO_GHQR_CT),
lambda modified: self._modified_file_callback(
modified, DataTypeKey.CTDMO_GHQR_CT),
self._exception_callback)
if ct_harvester is not None:
harvesters.append(ct_harvester)
else:
log.warn('Could not build ctdmo_ghqr_ct harvester')
else:
log.warn(
'No configuration for ctdmo_ghqr_ct harvester, not building')
#
# Verify that the CT Recovered harvester has been configured.
# If so, build the harvester and add it to the list of harvesters.
#
if DataTypeKey.CTDMO_GHQR_SIO_MULE in self._harvester_config:
ctdmo_ghqr_sio_mule_harvester = SingleFileHarvester(
self._harvester_config.get(DataTypeKey.CTDMO_GHQR_SIO_MULE),
driver_state[DataTypeKey.CTDMO_GHQR_SIO_MULE],
lambda file_state: self._file_changed_callback(
file_state, DataTypeKey.CTDMO_GHQR_SIO_MULE),
self._exception_callback)
if ctdmo_ghqr_sio_mule_harvester is not None:
harvesters.append(ctdmo_ghqr_sio_mule_harvester)
else:
log.warn('Could not build ctdmo_ghqr_sio_mule harvester')
else:
log.warn(
'No configuration for ctdmo_ghqr_sio_mule harvester, not building'
)
return harvesters