def init_queue(self): queue_location = self.conf.get( 'queue_location', '/tmp/oio-event-queue.db') self.queue = SqliteQueue('oio_event', queue_location)
class EventAgent(Daemon): def __init__(self, conf): validate_service_conf(conf) self.conf = conf self.logger = get_logger(conf) self.running = False self.retry_interval = int_value(conf.get('retry_interval'), 30) self.last_retry = 0 self.init_zmq() self.init_queue() self.init_workers() def run(self, *args, **kwargs): try: self.logger.info('event agent: starting') pool = GreenPool(len(self.workers)) for worker in self.workers: pool.spawn(worker.start) def front(server, backend): while True: msg = server.recv_multipart() if validate_msg(msg): try: event_id = sqlite3.Binary(msg[2]) data = msg[3] self.queue.put(event_id, data) event = ['', msg[2], msg[3]] backend.send_multipart(event) except Exception: pass finally: ack = msg[0:3] server.send_multipart(ack) def back(backend): while True: msg = backend.recv_multipart() event_id = msg[1] success = msg[2] event_id = sqlite3.Binary(event_id) if not success: self.queue.failed(event_id) else: self.queue.delete(event_id) boss_pool = GreenPool(2) boss_pool.spawn_n(front, self.server, self.backend) boss_pool.spawn_n(back, self.backend) while True: sleep(1) now = time.time() if now - self.last_retry > self.retry_interval: self.retry() self.last_retry = now for w in self.workers: if w.failed: self.workers.remove(w) self.logger.warn('restart worker "%s"', w.name) new_w = EventWorker(self.conf, w.name, self.context) self.workers.append(new_w) pool.spawn(new_w.start) except Exception as e: self.logger.error('ERROR in main loop %s', e) raise e finally: self.logger.warn('event agent: stopping') self.stop_workers() def init_zmq(self): self.context = zmq.Context() self.server = self.context.socket(zmq.ROUTER) bind_addr = self.conf.get('bind_addr', 'ipc:///tmp/run/event-agent.sock') self.server.bind(bind_addr) self.backend = self.context.socket(zmq.DEALER) self.backend.bind('inproc://event-front') def init_queue(self): queue_location = self.conf.get( 'queue_location', '/tmp/oio-event-queue.db') self.queue = SqliteQueue('oio_event', queue_location) def init_workers(self): nbworkers = int_value(self.conf.get('workers'), 2) workers = [] for i in xrange(nbworkers): workers.append(EventWorker(self.conf, str(i), self.context)) self.workers = workers def stop_workers(self): for worker in self.workers: worker.stop() def retry(self): cursor = self.queue.load(EVENT_BATCH_SIZE) for event in cursor: event_id, data = event msg = ['', event_id, str(data)] self.backend.send_multipart(msg)
def init_queue(self): queue_location = self.conf.get('queue_location', '/tmp/oio-event-queue.db') self.queue = SqliteQueue('oio_event', queue_location)
def init_queue(self): queue_location = self.conf.get("queue_location", "/tmp/oio-event-queue.db") self.queue = SqliteQueue("oio_event", queue_location)
class EventAgent(Daemon): def __init__(self, conf): validate_service_conf(conf) self.conf = conf self.logger = get_logger(conf) self.running = False self.retry_interval = int_value(conf.get('retry_interval'), 30) self.last_retry = 0 self.init_zmq() self.init_queue() self.init_workers() def run(self, *args, **kwargs): try: self.logger.info('event agent: starting') pool = GreenPool(len(self.workers)) for worker in self.workers: pool.spawn(worker.start) def front(server, backend): while True: msg = server.recv_multipart() if validate_msg(msg): try: event_id = sqlite3.Binary(msg[2]) data = msg[3] self.queue.put(event_id, data) event = ['', msg[2], msg[3]] backend.send_multipart(event) except Exception: pass finally: ack = msg[0:3] server.send_multipart(ack) def back(backend): while True: msg = backend.recv_multipart() event_id = msg[1] success = msg[2] event_id = sqlite3.Binary(event_id) if not success: self.queue.failed(event_id) else: self.queue.delete(event_id) boss_pool = GreenPool(2) boss_pool.spawn_n(front, self.server, self.backend) boss_pool.spawn_n(back, self.backend) while True: sleep(1) now = time.time() if now - self.last_retry > self.retry_interval: self.retry() self.last_retry = now for w in self.workers: if w.failed: self.workers.remove(w) self.logger.warn('restart worker "%s"', w.name) new_w = EventWorker(self.conf, w.name, self.context) self.workers.append(new_w) pool.spawn(new_w.start) except Exception as e: self.logger.error('ERROR in main loop %s', e) raise e finally: self.logger.warn('event agent: stopping') self.stop_workers() def init_zmq(self): self.context = zmq.Context() self.server = self.context.socket(zmq.ROUTER) bind_addr = self.conf.get('bind_addr', 'ipc:///tmp/run/event-agent.sock') self.server.bind(bind_addr) self.backend = self.context.socket(zmq.DEALER) self.backend.bind('inproc://event-front') def init_queue(self): queue_location = self.conf.get('queue_location', '/tmp/oio-event-queue.db') self.queue = SqliteQueue('oio_event', queue_location) def init_workers(self): nbworkers = int_value(self.conf.get('workers'), 2) workers = [] for i in xrange(nbworkers): workers.append(EventWorker(self.conf, str(i), self.context)) self.workers = workers def stop_workers(self): for worker in self.workers: worker.stop() def retry(self): cursor = self.queue.load(EVENT_BATCH_SIZE) for event in cursor: event_id, data = event msg = ['', event_id, str(data)] self.backend.send_multipart(msg)