class Persist(Component): channel = 'persist' def __init__(self, pool_count=10): super(Persist, self).__init__() self._pool_count = pool_count self.queue = Queue() self.spool = Spooler().register(self) def started(self, *args, **kwargs): self.hostname = self.root.config.get('core', 'hostname') b = self.root.config.get('core', 'persist_backend') b = b.lower() h = self.root.config.get('backend:%s' % b, 'host') p = self.root.config.get('backend:%s' % b, 'port') if b in ('http', 'https'): self._backends = backends = {} self._backend_state = {} while len(self._backends) < self._pool_count: bid = uuid4().hex backends[bid] = RetryHTTPClient(host=h, port=p, scheme=b, channel=bid).register(self) self._backend_state[bid] = IDLE else: logger.error("Invalid backend given: %s" % b) raise SystemExit Timer(CHECK_INTERVAL, Event(), 'persist', t=self, persist=True).register(self) @handler('response_success') def _on_response(self, bid): rid = self._backend_state[bid] self.fire(Event(rid), 'remove', target=self.spool) self._backend_state[bid] = IDLE self.fire(Event(bid), 'persist', target=self) def persist(self, bid=None, rid=None): if rid is None and self.queue.empty(): logger.debug("Nothing to persist, exiting persist.persist") #nothing to persist. return if bid is None: available_backends = [b for b,s in self._backend_state.iteritems() if s is IDLE] if not available_backends: if rid is not None: self.queue.put(rid[0]) #sleep and try again? logger.debug("No available backends, exiting persist.persist") return bid = available_backends[0] if rid is None: rid = self.queue.get() (sourcetype, timestamp, data, extra) = self.spool.get(rid) else: (sourcetype, timestamp, data, extra) = rid[1] rid = rid[0] self._backend_state[bid] = rid #self.fire(Event(), 'connect', target=bid) self.fire(request_factory( self.hostname, sourcetype, timestamp, data, extra), 'request', target=bid )
class Persist(Component): channel = 'persist' def __init__(self, pool_count=10): super(Persist, self).__init__() self._pool_count = pool_count self.queue = Queue() self.spool = Spooler().register(self) def started(self, *args, **kwargs): self.hostname = self.root.config.get('core', 'hostname') b = self.root.config.get('core', 'persist_backend') b = b.lower() h = self.root.config.get('backend:%s' % b, 'host') p = self.root.config.get('backend:%s' % b, 'port') if b in ('http', 'https'): self._backends = backends = {} self._backend_state = {} while len(self._backends) < self._pool_count: bid = uuid4().hex backends[bid] = RetryHTTPClient(host=h, port=p, scheme=b, channel=bid).register(self) self._backend_state[bid] = IDLE else: logger.error("Invalid backend given: %s" % b) raise SystemExit Timer(CHECK_INTERVAL, Event(), 'persist', t=self, persist=True).register(self) @handler('response_success') def _on_response(self, bid): rid = self._backend_state[bid] self.fire(Event(rid), 'remove', target=self.spool) self._backend_state[bid] = IDLE self.fire(Event(bid), 'persist', target=self) def persist(self, bid=None, rid=None): if rid is None and self.queue.empty(): logger.debug("Nothing to persist, exiting persist.persist") #nothing to persist. return if bid is None: available_backends = [ b for b, s in self._backend_state.iteritems() if s is IDLE ] if not available_backends: if rid is not None: self.queue.put(rid[0]) #sleep and try again? logger.debug("No available backends, exiting persist.persist") return bid = available_backends[0] if rid is None: rid = self.queue.get() (sourcetype, timestamp, data, extra) = self.spool.get(rid) else: (sourcetype, timestamp, data, extra) = rid[1] rid = rid[0] self._backend_state[bid] = rid #self.fire(Event(), 'connect', target=bid) self.fire(request_factory(self.hostname, sourcetype, timestamp, data, extra), 'request', target=bid)