def init_queue(self):
queue_location = self.conf.get(
'queue_location', '/tmp/oio-event-queue.db')
self.queue = SqliteQueue('oio_event', queue_location)
class EventAgent(Daemon):
def __init__(self, conf):
validate_service_conf(conf)
self.conf = conf
self.logger = get_logger(conf)
self.running = False
self.retry_interval = int_value(conf.get('retry_interval'), 30)
self.last_retry = 0
self.init_zmq()
self.init_queue()
self.init_workers()
def run(self, *args, **kwargs):
try:
self.logger.info('event agent: starting')
pool = GreenPool(len(self.workers))
for worker in self.workers:
pool.spawn(worker.start)
def front(server, backend):
while True:
msg = server.recv_multipart()
if validate_msg(msg):
try:
event_id = sqlite3.Binary(msg[2])
data = msg[3]
self.queue.put(event_id, data)
event = ['', msg[2], msg[3]]
backend.send_multipart(event)
except Exception:
pass
finally:
ack = msg[0:3]
server.send_multipart(ack)
def back(backend):
while True:
msg = backend.recv_multipart()
event_id = msg[1]
success = msg[2]
event_id = sqlite3.Binary(event_id)
if not success:
self.queue.failed(event_id)
else:
self.queue.delete(event_id)
boss_pool = GreenPool(2)
boss_pool.spawn_n(front, self.server, self.backend)
boss_pool.spawn_n(back, self.backend)
while True:
sleep(1)
now = time.time()
if now - self.last_retry > self.retry_interval:
self.retry()
self.last_retry = now
for w in self.workers:
if w.failed:
self.workers.remove(w)
self.logger.warn('restart worker "%s"', w.name)
new_w = EventWorker(self.conf, w.name, self.context)
self.workers.append(new_w)
pool.spawn(new_w.start)
except Exception as e:
self.logger.error('ERROR in main loop %s', e)
raise e
finally:
self.logger.warn('event agent: stopping')
self.stop_workers()
def init_zmq(self):
self.context = zmq.Context()
self.server = self.context.socket(zmq.ROUTER)
bind_addr = self.conf.get('bind_addr',
'ipc:///tmp/run/event-agent.sock')
self.server.bind(bind_addr)
self.backend = self.context.socket(zmq.DEALER)
self.backend.bind('inproc://event-front')
def init_queue(self):
queue_location = self.conf.get(
'queue_location', '/tmp/oio-event-queue.db')
self.queue = SqliteQueue('oio_event', queue_location)
def init_workers(self):
nbworkers = int_value(self.conf.get('workers'), 2)
workers = []
for i in xrange(nbworkers):
workers.append(EventWorker(self.conf, str(i), self.context))
self.workers = workers
def stop_workers(self):
for worker in self.workers:
worker.stop()
def retry(self):
cursor = self.queue.load(EVENT_BATCH_SIZE)
for event in cursor:
event_id, data = event
msg = ['', event_id, str(data)]
self.backend.send_multipart(msg)
def init_queue(self):
queue_location = self.conf.get('queue_location',
'/tmp/oio-event-queue.db')
self.queue = SqliteQueue('oio_event', queue_location)
def init_queue(self):
queue_location = self.conf.get("queue_location", "/tmp/oio-event-queue.db")
self.queue = SqliteQueue("oio_event", queue_location)
class EventAgent(Daemon):
def __init__(self, conf):
validate_service_conf(conf)
self.conf = conf
self.logger = get_logger(conf)
self.running = False
self.retry_interval = int_value(conf.get('retry_interval'), 30)
self.last_retry = 0
self.init_zmq()
self.init_queue()
self.init_workers()
def run(self, *args, **kwargs):
try:
self.logger.info('event agent: starting')
pool = GreenPool(len(self.workers))
for worker in self.workers:
pool.spawn(worker.start)
def front(server, backend):
while True:
msg = server.recv_multipart()
if validate_msg(msg):
try:
event_id = sqlite3.Binary(msg[2])
data = msg[3]
self.queue.put(event_id, data)
event = ['', msg[2], msg[3]]
backend.send_multipart(event)
except Exception:
pass
finally:
ack = msg[0:3]
server.send_multipart(ack)
def back(backend):
while True:
msg = backend.recv_multipart()
event_id = msg[1]
success = msg[2]
event_id = sqlite3.Binary(event_id)
if not success:
self.queue.failed(event_id)
else:
self.queue.delete(event_id)
boss_pool = GreenPool(2)
boss_pool.spawn_n(front, self.server, self.backend)
boss_pool.spawn_n(back, self.backend)
while True:
sleep(1)
now = time.time()
if now - self.last_retry > self.retry_interval:
self.retry()
self.last_retry = now
for w in self.workers:
if w.failed:
self.workers.remove(w)
self.logger.warn('restart worker "%s"', w.name)
new_w = EventWorker(self.conf, w.name, self.context)
self.workers.append(new_w)
pool.spawn(new_w.start)
except Exception as e:
self.logger.error('ERROR in main loop %s', e)
raise e
finally:
self.logger.warn('event agent: stopping')
self.stop_workers()
def init_zmq(self):
self.context = zmq.Context()
self.server = self.context.socket(zmq.ROUTER)
bind_addr = self.conf.get('bind_addr',
'ipc:///tmp/run/event-agent.sock')
self.server.bind(bind_addr)
self.backend = self.context.socket(zmq.DEALER)
self.backend.bind('inproc://event-front')
def init_queue(self):
queue_location = self.conf.get('queue_location',
'/tmp/oio-event-queue.db')
self.queue = SqliteQueue('oio_event', queue_location)
def init_workers(self):
nbworkers = int_value(self.conf.get('workers'), 2)
workers = []
for i in xrange(nbworkers):
workers.append(EventWorker(self.conf, str(i), self.context))
self.workers = workers
def stop_workers(self):
for worker in self.workers:
worker.stop()
def retry(self):
cursor = self.queue.load(EVENT_BATCH_SIZE)
for event in cursor:
event_id, data = event
msg = ['', event_id, str(data)]
self.backend.send_multipart(msg)
