class EventFeeder(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.queueConfigMapper = queue_config_mapper self.communicator = communicator self.pluginFactory = PluginFactory() # main loop def run(self): lockedBy = 'eventfeeder-{0}'.format(self.ident) while True: mainLog = self.make_logger(_logger, 'id={0}'.format(lockedBy), method_name='run') mainLog.debug('getting workers to feed events') workSpecsPerQueue = self.dbProxy.get_workers_to_feed_events(harvester_config.eventfeeder.maxWorkers, harvester_config.eventfeeder.lockInterval) mainLog.debug('got {0} queues'.format(len(workSpecsPerQueue))) # loop over all workers for queueName, workSpecList in iteritems(workSpecsPerQueue): tmpQueLog = self.make_logger(_logger, 'queue={0}'.format(queueName), method_name='run') # check queue if not self.queueConfigMapper.has_queue(queueName): tmpQueLog.error('config not found') continue # get queue queueConfig = self.queueConfigMapper.get_queue(queueName) # get plugin messenger = self.pluginFactory.get_plugin(queueConfig.messenger) # loop over all workers for workSpec in workSpecList: tmpLog = core_utils.make_logger(_logger, 'workerID={0}'.format(workSpec.workerID), method_name='run') # get events tmpLog.debug('get events') tmpStat, events = self.communicator.get_event_ranges(workSpec.eventsRequestParams) # failed if tmpStat is False: tmpLog.error('failed to get events with {0}'.format(events)) continue tmpStat = messenger.feed_events(workSpec, events) # failed if tmpStat is False: tmpLog.error('failed to feed events') continue # update worker workSpec.eventsRequest = WorkSpec.EV_useEvents workSpec.eventsRequestParams = None workSpec.eventFeedTime = None # update local database tmpStat = self.dbProxy.update_worker(workSpec) tmpLog.debug('done with {0}'.format(tmpStat)) tmpQueLog.debug('done') mainLog.debug('done') # check if being terminated if self.terminated(harvester_config.eventfeeder.sleepTime): mainLog.debug('terminated') return
class EventFeeder(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.queueConfigMapper = queue_config_mapper self.communicator = communicator self.pluginFactory = PluginFactory() # main loop def run(self): lockedBy = 'eventfeeder-{0}'.format(self.get_pid()) while True: mainLog = self.make_logger(_logger, 'id={0}'.format(lockedBy), method_name='run') mainLog.debug('getting workers to feed events') workSpecsPerQueue = self.dbProxy.get_workers_to_feed_events(harvester_config.eventfeeder.maxWorkers, harvester_config.eventfeeder.lockInterval, lockedBy) mainLog.debug('got {0} queues'.format(len(workSpecsPerQueue))) # loop over all workers for queueName, workSpecList in iteritems(workSpecsPerQueue): tmpQueLog = self.make_logger(_logger, 'queue={0}'.format(queueName), method_name='run') # check queue if not self.queueConfigMapper.has_queue(queueName): tmpQueLog.error('config not found') continue # get queue queueConfig = self.queueConfigMapper.get_queue(queueName) if hasattr(queueConfig, 'scatteredEvents') and queueConfig.scatteredEvents: scattered = True else: scattered = False # get plugin messenger = self.pluginFactory.get_plugin(queueConfig.messenger) # loop over all workers for workSpec in workSpecList: tmpLog = core_utils.make_logger(_logger, 'workerID={0}'.format(workSpec.workerID), method_name='run') # lock worker again lockedFlag = self.dbProxy.lock_worker_again_to_feed_events(workSpec.workerID, lockedBy) if not lockedFlag: tmpLog.debug('skipped since locked by another') continue # get events tmpLog.debug('get events') tmpStat, events = self.communicator.get_event_ranges(workSpec.eventsRequestParams, scattered, workSpec.get_access_point()) # failed if tmpStat is False: tmpLog.error('failed to get events with {0}'.format(events)) continue # lock worker again lockedFlag = self.dbProxy.lock_worker_again_to_feed_events(workSpec.workerID, lockedBy) if not lockedFlag: tmpLog.debug('skipped before feeding since locked by another') continue tmpStat = messenger.feed_events(workSpec, events) # failed if tmpStat is False: tmpLog.error('failed to feed events') continue # dump for pandaID, eventList in iteritems(events): try: nRanges = workSpec.eventsRequestParams[pandaID]['nRanges'] except Exception: nRanges = None tmpLog.debug('got {0} events for PandaID={1} while getting {2} events'.format(len(eventList), pandaID, nRanges)) # disable multi workers if workSpec.mapType == WorkSpec.MT_MultiWorkers: if len(eventList) == 0 or (nRanges is not None and len(eventList) < nRanges): tmpStat = self.dbProxy.disable_multi_workers(pandaID) if tmpStat == 1: tmpStr = 'disabled MultiWorkers for PandaID={0}'.format(pandaID) tmpLog.debug(tmpStr) # update worker workSpec.eventsRequest = WorkSpec.EV_useEvents workSpec.eventsRequestParams = None workSpec.eventFeedTime = None workSpec.eventFeedLock = None # update local database tmpStat = self.dbProxy.update_worker(workSpec, {'eventFeedLock': lockedBy}) tmpLog.debug('done with {0}'.format(tmpStat)) tmpQueLog.debug('done') mainLog.debug('done') # check if being terminated if self.terminated(harvester_config.eventfeeder.sleepTime): mainLog.debug('terminated') return
class EventFeeder(AgentBase): # constructor def __init__(self, communicator, queue_config_mapper, single_mode=False): AgentBase.__init__(self, single_mode) self.dbProxy = DBProxy() self.queueConfigMapper = queue_config_mapper self.communicator = communicator self.pluginFactory = PluginFactory() # main loop def run(self): lockedBy = 'eventfeeder-{0}'.format(self.get_pid()) while True: mainLog = self.make_logger(_logger, 'id={0}'.format(lockedBy), method_name='run') mainLog.debug('getting workers to feed events') workSpecsPerQueue = self.dbProxy.get_workers_to_feed_events(harvester_config.eventfeeder.maxWorkers, harvester_config.eventfeeder.lockInterval, lockedBy) mainLog.debug('got {0} queues'.format(len(workSpecsPerQueue))) # loop over all workers for queueName, workSpecList in iteritems(workSpecsPerQueue): tmpQueLog = self.make_logger(_logger, 'queue={0}'.format(queueName), method_name='run') # check queue if not self.queueConfigMapper.has_queue(queueName): tmpQueLog.error('config not found') continue # get queue queueConfig = self.queueConfigMapper.get_queue(queueName) if hasattr(queueConfig, 'scatteredEvents') and queueConfig.scatteredEvents: scattered = True else: scattered = False # get plugin messenger = self.pluginFactory.get_plugin(queueConfig.messenger) # loop over all workers for workSpec in workSpecList: tmpLog = core_utils.make_logger(_logger, 'workerID={0}'.format(workSpec.workerID), method_name='run') # lock worker again lockedFlag = self.dbProxy.lock_worker_again_to_feed_events(workSpec.workerID, lockedBy) if not lockedFlag: tmpLog.debug('skipped since locked by another') continue # get events tmpLog.debug('get events') tmpStat, events = self.communicator.get_event_ranges(workSpec.eventsRequestParams, scattered) # failed if tmpStat is False: tmpLog.error('failed to get events with {0}'.format(events)) continue # lock worker again lockedFlag = self.dbProxy.lock_worker_again_to_feed_events(workSpec.workerID, lockedBy) if not lockedFlag: tmpLog.debug('skipped before feeding since locked by another') continue tmpStat = messenger.feed_events(workSpec, events) # failed if tmpStat is False: tmpLog.error('failed to feed events') continue # dump for pandaID, eventList in iteritems(events): try: nRanges = workSpec.eventsRequestParams[pandaID]['nRanges'] except Exception: nRanges = None tmpLog.debug('got {0} events for PandaID={1} while getting {2} events'.format(len(eventList), pandaID, nRanges)) # disable multi workers if workSpec.mapType == WorkSpec.MT_MultiWorkers: if len(eventList) == 0 or (nRanges is not None and len(eventList) < nRanges): tmpStat = self.dbProxy.disable_multi_workers(pandaID) if tmpStat == 1: tmpStr = 'disabled MultiWorkers for PandaID={0}'.format(pandaID) tmpLog.debug(tmpStr) # update worker workSpec.eventsRequest = WorkSpec.EV_useEvents workSpec.eventsRequestParams = None workSpec.eventFeedTime = None workSpec.eventFeedLock = None # update local database tmpStat = self.dbProxy.update_worker(workSpec, {'eventFeedLock': lockedBy}) tmpLog.debug('done with {0}'.format(tmpStat)) tmpQueLog.debug('done') mainLog.debug('done') # check if being terminated if self.terminated(harvester_config.eventfeeder.sleepTime): mainLog.debug('terminated') return