def run(self):
logger.debug("initializing spool")
config = self.config_parser
self.spool = spool = Spooler(config)
num_gather_workers = None
if config.has_option('core', 'gather_workers'):
num_gather_workers = abs(config.getint('core', 'gather_workers'))
if not num_gather_workers:
num_gather_workers = GATHER_POOL_WORKERS
self._gather_pool = pool = Pool(num_gather_workers)
persist_queue = self.persist_queue
self.collect_manager = CollectPlugins(
base_class=Collect,
config=config,
init_args=(config, persist_queue, spool, pool),
entry_points='arke_plugins',
)
self.collect_manager.load(pool=self._gather_pool)
try:
self.persist_runner()
except KeyboardInterrupt:
pass
self.shutdown()
def run(self):
logging.debug("initializing spool")
self.spool = spool = Spooler(self.config_parser)
self._gather_pool = pool = Pool(GATHER_POOL_WORKERS)
config = self.config_parser
persist_queue = self.persist_queue
self.collect_manager = CollectPlugins(
base_class=Collect,
config=config,
init_args=(config, persist_queue, spool, pool),
entry_points='arke_plugins',
)
self.collect_manager.load(pool=self._gather_pool)
try:
self.persist_runner()
except KeyboardInterrupt:
pass
self.shutdown()
def __init__(self, pool_count=10):
super(Persist, self).__init__()
self._pool_count = pool_count
self.queue = Queue()
self.spool = Spooler().register(self)
class Persist(Component):
channel = 'persist'
def __init__(self, pool_count=10):
super(Persist, self).__init__()
self._pool_count = pool_count
self.queue = Queue()
self.spool = Spooler().register(self)
def started(self, *args, **kwargs):
self.hostname = self.root.config.get('core', 'hostname')
b = self.root.config.get('core', 'persist_backend')
b = b.lower()
h = self.root.config.get('backend:%s' % b, 'host')
p = self.root.config.get('backend:%s' % b, 'port')
if b in ('http', 'https'):
self._backends = backends = {}
self._backend_state = {}
while len(self._backends) < self._pool_count:
bid = uuid4().hex
backends[bid] = RetryHTTPClient(host=h, port=p, scheme=b, channel=bid).register(self)
self._backend_state[bid] = IDLE
else:
logger.error("Invalid backend given: %s" % b)
raise SystemExit
Timer(CHECK_INTERVAL, Event(), 'persist', t=self, persist=True).register(self)
@handler('response_success')
def _on_response(self, bid):
rid = self._backend_state[bid]
self.fire(Event(rid), 'remove', target=self.spool)
self._backend_state[bid] = IDLE
self.fire(Event(bid), 'persist', target=self)
def persist(self, bid=None, rid=None):
if rid is None and self.queue.empty():
logger.debug("Nothing to persist, exiting persist.persist")
#nothing to persist.
return
if bid is None:
available_backends = [b for b,s in self._backend_state.iteritems() if s is IDLE]
if not available_backends:
if rid is not None:
self.queue.put(rid[0])
#sleep and try again?
logger.debug("No available backends, exiting persist.persist")
return
bid = available_backends[0]
if rid is None:
rid = self.queue.get()
(sourcetype, timestamp, data, extra) = self.spool.get(rid)
else:
(sourcetype, timestamp, data, extra) = rid[1]
rid = rid[0]
self._backend_state[bid] = rid
#self.fire(Event(), 'connect', target=bid)
self.fire(request_factory(
self.hostname, sourcetype, timestamp, data, extra),
'request', target=bid
)
def __init__(self, pool_count=10):
super(Persist, self).__init__()
self._pool_count = pool_count
self.queue = Queue()
self.spool = Spooler().register(self)
class Persist(Component):
channel = 'persist'
def __init__(self, pool_count=10):
super(Persist, self).__init__()
self._pool_count = pool_count
self.queue = Queue()
self.spool = Spooler().register(self)
def started(self, *args, **kwargs):
self.hostname = self.root.config.get('core', 'hostname')
b = self.root.config.get('core', 'persist_backend')
b = b.lower()
h = self.root.config.get('backend:%s' % b, 'host')
p = self.root.config.get('backend:%s' % b, 'port')
if b in ('http', 'https'):
self._backends = backends = {}
self._backend_state = {}
while len(self._backends) < self._pool_count:
bid = uuid4().hex
backends[bid] = RetryHTTPClient(host=h,
port=p,
scheme=b,
channel=bid).register(self)
self._backend_state[bid] = IDLE
else:
logger.error("Invalid backend given: %s" % b)
raise SystemExit
Timer(CHECK_INTERVAL, Event(), 'persist', t=self,
persist=True).register(self)
@handler('response_success')
def _on_response(self, bid):
rid = self._backend_state[bid]
self.fire(Event(rid), 'remove', target=self.spool)
self._backend_state[bid] = IDLE
self.fire(Event(bid), 'persist', target=self)
def persist(self, bid=None, rid=None):
if rid is None and self.queue.empty():
logger.debug("Nothing to persist, exiting persist.persist")
#nothing to persist.
return
if bid is None:
available_backends = [
b for b, s in self._backend_state.iteritems() if s is IDLE
]
if not available_backends:
if rid is not None:
self.queue.put(rid[0])
#sleep and try again?
logger.debug("No available backends, exiting persist.persist")
return
bid = available_backends[0]
if rid is None:
rid = self.queue.get()
(sourcetype, timestamp, data, extra) = self.spool.get(rid)
else:
(sourcetype, timestamp, data, extra) = rid[1]
rid = rid[0]
self._backend_state[bid] = rid
#self.fire(Event(), 'connect', target=bid)
self.fire(request_factory(self.hostname, sourcetype, timestamp, data,
extra),
'request',
target=bid)
