rt_key=a_key2) def master_alarm(): print('master alarm ! All publishers disconnected') cl = Client('localhost:15672', 'user1', 'password1') if cl.is_alive(): cl.create_queue(vhost=a_vhost, name=a_Q1, durable=True) cl.create_queue(vhost=a_vhost, name=a_Q2, durable=True) try: cl.delete_binding(vhost=a_vhost, exchange=a_exchange, queue=a_Q1, rt_key=a_key2) except http.HTTPError: print("no binding " + pub2 + " to " + a_Q1) try: cl.delete_binding(vhost=a_vhost, exchange=a_exchange, queue=a_Q2, rt_key=a_key1) except http.HTTPError: print("no binding " + pub1 + " to " + a_Q2) cl.create_binding(vhost=a_vhost, exchange=a_exchange, queue=a_Q1,
class Collector(object): def __init__(self, index): self.index = index self.cancelled = False self.transport = self.protocol = None self.messages = [] self.block_size = 1000 self.exchange = "logstash" self.current_date = None self.logger = getLogger('alco.collector.%s' % self.index.name) self.amqp = self.redis = self.conn = self.vhost = self.rabbit = None self.insert_thread = None self.query_queue = queue.Queue() self.result_queue = queue.Queue() self.result_queue.put(None) self.query = self.values_stub = None self.existing = self.included = self.indexed = self.filtered = None def cancel(self): self.cancelled = True def inserter_loop(self): self.conn = connections[ALCO_SETTINGS['SPHINX_DATABASE_NAME']] while not self.cancelled: try: query, args = self.query_queue.get(block=True, timeout=1) except queue.Empty: continue result = self.insert_data(query, args) self.result_queue.put(result) def connect(self): connections['default'].close() rabbitmq = ALCO_SETTINGS['RABBITMQ'] self.amqp = Connection(**rabbitmq) self.redis = redis.Redis(**ALCO_SETTINGS['REDIS']) self.insert_thread = Thread(target=self.inserter_loop) self.insert_thread.start() hostname = '%s:%s' % (rabbitmq['host'], ALCO_SETTINGS['RABBITMQ_API_PORT']) self.rabbit = Client(hostname, rabbitmq['userid'], rabbitmq['password']) self.vhost = rabbitmq['virtual_host'] # noinspection PyUnusedLocal def process_sigint(self, signum, frame): self.logger.info("Got signal %s" % signum) self.cancel() self.logger.info("Futures cancelled, wait for thread") self.insert_thread.join() self.logger.info("Thread done") def __call__(self): signal.signal(signal.SIGINT, self.process_sigint) signal.signal(signal.SIGTERM, self.process_sigint) try: self.logger.debug("Connecting to RabbitMQ") self.connect() self.declare_queue() self.cleanup_bindings() channel = self.amqp.channel() channel.basic_qos(0, 1000, False) channel.basic_consume(self.index.queue_name, callback=self.process_message, no_ack=True) start = time.time() self.logger.debug("Start processing messages") while not self.cancelled: try: self.amqp.drain_events(timeout=1) except (socket.timeout, OSError): pass if time.time() - start > 1: self.push_messages() start = time.time() except KeyboardInterrupt: self.logger.warning("Got SIGINT, exit(0)") finally: self.amqp.close() sys.exit(0) def process_message(self, msg): data = json.loads(six.binary_type(msg.body)) ts = data.pop('@timestamp') data.pop("@version") msg = data.pop('message') seq = data.pop('seq', 0) dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") result = { 'ts': time.mktime(dt.timetuple()), 'ms': dt.microsecond, 'seq': seq, 'message': msg, 'data': data } self.messages.append(result) d = dt.date() if not self.current_date: self.current_date = d if d != self.current_date: self.current_date = d self.push_messages() if len(self.messages) >= self.block_size: self.push_messages() def declare_queue(self): channel = self.amqp.channel() """:type channel: amqp.channel.Channel""" durable = self.index.durable channel.exchange_declare(exchange=self.exchange, type='topic', durable=durable, auto_delete=False) channel.queue_declare(self.index.queue_name, durable=durable, auto_delete=False) for rk in self.get_routing_keys(): channel.queue_bind(self.index.queue_name, exchange=self.exchange, routing_key=rk) def get_routing_keys(self): return map(lambda x: x.strip(), self.index.routing_key.split(',')) def cleanup_bindings(self): self.logger.debug("Checking bindings") queue = self.index.queue_name exchange = self.exchange bindings = self.rabbit.get_queue_bindings(self.vhost, queue) bindings = [b for b in bindings if b['source'] == exchange] allowed = self.get_routing_keys() q = six.moves.urllib.parse.quote for b in bindings: rk = b['routing_key'] if rk in allowed: continue self.logger.debug("Unbind %s with RK=%s" % (queue, rk)) self.rabbit.delete_binding(self.vhost, exchange, q(queue), q(rk)) def push_messages(self): try: request_started.send(None, environ=None) self._push_messages() except Exception as e: self.logger.exception(e) raise finally: request_finished.send(None) def _push_messages(self): messages, self.messages = self.messages, [] if not messages: return message_count = len(messages) self.logger.info("Saving %s events" % message_count) columns = defaultdict(set) suffix = self.current_date.strftime("%Y%m%d") name = "%s_%s" % (self.index.name, suffix) args = [] self.load_index_columns() self.prepare_query(name) pkeys = self.get_primary_keys(messages) seen = set() for pk, data in zip(pkeys, messages): # saving seen columns to LoggerColumn model, collecting unique # values for caching in redis js = data['data'] self.process_js_columns(js, columns, self.included, seen) js_str = json.dumps(js) values = tuple(js.get(c) or '' for c in self.indexed) args.extend((pk, js_str, data['message']) + values) query = self.query + ','.join([self.values_stub] * message_count) self.save_column_values(columns) self.save_new_columns(seen) if self.result_queue.empty(): self.logger.debug("Insert still running, waiting") while not self.cancelled: try: self.result_queue.get(block=True, timeout=1) except queue.Empty: continue self.query_queue.put((query, args)) def insert_data(self, query, args): self.logger.debug("Inserting logs to searchd") result = None for _ in 1, 2, 3: try: c = self.conn.cursor() result = c.execute(query, args) self.logger.debug("%s rows inserted" % c.rowcount) c.close() except ProgrammingError: self.logger.exception( "Can't insert values to index: %s" % query) except DatabaseError as e: self.logger.exception("Sphinx connection error: %s" % e) try: close_old_connections() except Exception as e: self.logger.exception("Can't reconnect: %s" % e) os.kill(os.getpid(), signal.SIGKILL) except Exception: self.logger.exception("Unhandled error in insert_data") else: return result self.logger.error("Can't insert data in 3 tries, exit process") os.kill(os.getpid(), signal.SIGKILL) def save_new_columns(self, seen): self.logger.debug("Check for new columns") for column in seen - set(self.existing): self.logger.debug("Register column %s" % column) self.index.loggercolumn_set.create(name=column) def save_column_values(self, columns): self.logger.debug("Saving values for filtered columns") ts = time.time() for column in self.filtered: values = columns.get(column) if not values: continue key = keys.KEY_COLUMN_VALUES.format(index=self.index.name, column=column) values = {v: ts for v in values} self.redis.zadd(key, **values) def prepare_query(self, name): if self.indexed: self.query = "REPLACE INTO %s (id, js, logline, %s) VALUES " % ( name, ', '.join(self.indexed)) else: self.query = "REPLACE INTO %s (id, js, logline) VALUES " % name sql_col_count = len(self.indexed) + 3 # + jd, js, logline self.values_stub = "(%s)" % ", ".join(["%s"] * sql_col_count) def load_index_columns(self): # all defined columns all_columns = list(self.index.loggercolumn_set.all()) included_columns = [c for c in all_columns if not c.excluded] filtered_columns = [c for c in included_columns if c.filtered] indexed_columns = [c for c in included_columns if c.indexed] self.existing = [c.name for c in all_columns] self.included = [c.name for c in included_columns] self.filtered = [c.name for c in filtered_columns] self.indexed = [c.name for c in indexed_columns] @staticmethod def process_js_columns(js, columns, included, seen): for key, value in list(js.items()): if key in ('pk', 'id', 'ts', 'ms', 'seq', 'model'): # escape fields reserved by Django and ALCO js['%s_x' % key] = js.pop(key) key = '%s_x' % key # save seen columns set if key not in seen: seen.add(key) if key not in included: # discard fields excluded from indexing js.pop(key) continue # save column values set if type(value) not in (bool, int, float, six.text_type): continue if value not in columns[key]: columns[key].add(value) def get_primary_keys(self, messages): """ Generate PK sequence for a list of messages.""" pkeys = [] pk = None for msg in messages: # pk is [timestamp][microseconds][randint] in 10based integer pk = int((msg['ts'] * 10**6 + msg['ms']) * 1000) + randint(0, 1000) pkeys.append(pk) self.logger.debug("first pk is %s" % pk) return pkeys
class Collector(object): def __init__(self, index): self.index = index self.cancelled = False self.transport = self.protocol = None self.messages = [] self.block_size = 1000 self.exchange = "logstash" self.current_date = None self.logger = getLogger('alco.collector.%s' % self.index.name) self.amqp = self.redis = self.conn = self.vhost = self.rabbit = None self.insert_thread = None self.query_queue = queue.Queue() self.result_queue = queue.Queue() self.result_queue.put(None) self.query = self.values_stub = None self.existing = self.included = self.indexed = self.filtered = None def cancel(self): self.cancelled = True def inserter_loop(self): self.conn = connections[ALCO_SETTINGS['SPHINX_DATABASE_NAME']] while not self.cancelled: try: query, args = self.query_queue.get(block=True, timeout=1) except queue.Empty: continue result = self.insert_data(query, args) self.result_queue.put(result) def connect(self): connections['default'].close() rabbitmq = ALCO_SETTINGS['RABBITMQ'] self.amqp = Connection(**rabbitmq) self.redis = redis.Redis(**ALCO_SETTINGS['REDIS']) self.insert_thread = Thread(target=self.inserter_loop) self.insert_thread.start() hostname = '%s:%s' % (rabbitmq['host'], ALCO_SETTINGS['RABBITMQ_API_PORT']) self.rabbit = Client(hostname, rabbitmq['userid'], rabbitmq['password']) self.vhost = rabbitmq['virtual_host'] # noinspection PyUnusedLocal def process_sigint(self, signum, frame): self.logger.info("Got signal %s" % signum) self.cancel() self.logger.info("Futures cancelled, wait for thread") self.insert_thread.join() self.logger.info("Thread done") def __call__(self): signal.signal(signal.SIGINT, self.process_sigint) signal.signal(signal.SIGTERM, self.process_sigint) try: self.logger.debug("Connecting to RabbitMQ") self.connect() self.declare_queue() self.cleanup_bindings() channel = self.amqp.channel() channel.basic_qos(0, 1000, False) channel.basic_consume(self.index.queue_name, callback=self.process_message, no_ack=True) start = time.time() self.logger.debug("Start processing messages") while not self.cancelled: try: self.amqp.drain_events(timeout=1) except (socket.timeout, OSError): pass if time.time() - start > 1: self.push_messages() start = time.time() except KeyboardInterrupt: self.logger.warning("Got SIGINT, exit(0)") finally: self.amqp.close() sys.exit(0) def process_message(self, msg): data = json.loads(six.binary_type(msg.body)) ts = data.pop('@timestamp') data.pop("@version") msg = data.pop('message') seq = data.pop('seq', 0) dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ") result = { 'ts': time.mktime(dt.timetuple()), 'ms': dt.microsecond, 'seq': seq, 'message': msg, 'data': data } self.messages.append(result) d = dt.date() if not self.current_date: self.current_date = d if d != self.current_date: self.current_date = d self.push_messages() if len(self.messages) >= self.block_size: self.push_messages() def declare_queue(self): channel = self.amqp.channel() """:type channel: amqp.channel.Channel""" durable = self.index.durable channel.exchange_declare(exchange=self.exchange, type='topic', durable=durable, auto_delete=False) channel.queue_declare(self.index.queue_name, durable=durable, auto_delete=False) for rk in self.get_routing_keys(): channel.queue_bind(self.index.queue_name, exchange=self.exchange, routing_key=rk) def get_routing_keys(self): return map(lambda x: x.strip(), self.index.routing_key.split(',')) def cleanup_bindings(self): self.logger.debug("Checking bindings") queue = self.index.queue_name exchange = self.exchange bindings = self.rabbit.get_queue_bindings(self.vhost, queue) bindings = [b for b in bindings if b['source'] == exchange] allowed = self.get_routing_keys() q = six.moves.urllib.parse.quote for b in bindings: rk = b['routing_key'] if rk in allowed: continue self.logger.debug("Unbind %s with RK=%s" % (queue, rk)) self.rabbit.delete_binding(self.vhost, exchange, q(queue), q(rk)) def push_messages(self): try: request_started.send(None, environ=None) self._push_messages() except Exception as e: self.logger.exception(e) raise finally: request_finished.send(None) def _push_messages(self): messages, self.messages = self.messages, [] if not messages: return message_count = len(messages) self.logger.info("Saving %s events" % message_count) columns = defaultdict(set) suffix = self.current_date.strftime("%Y%m%d") name = "%s_%s" % (self.index.name, suffix) args = [] self.load_index_columns() self.prepare_query(name) pkeys = self.get_primary_keys(messages) seen = set() for pk, data in zip(pkeys, messages): # saving seen columns to LoggerColumn model, collecting unique # values for caching in redis js = data['data'] self.process_js_columns(js, columns, self.included, seen) js_str = json.dumps(js) values = tuple(js.get(c) or '' for c in self.indexed) args.extend((pk, js_str, data['message']) + values) query = self.query + ','.join([self.values_stub] * message_count) self.save_column_values(columns) self.save_new_columns(seen) if self.result_queue.empty(): self.logger.debug("Insert still running, waiting") while not self.cancelled: try: self.result_queue.get(block=True, timeout=1) except queue.Empty: continue self.query_queue.put((query, args)) def insert_data(self, query, args): self.logger.debug("Inserting logs to searchd") result = None for _ in 1, 2, 3: try: c = self.conn.cursor() result = c.execute(query, args) self.logger.debug("%s rows inserted" % c.rowcount) c.close() except ProgrammingError: self.logger.exception("Can't insert values to index: %s" % query) except DatabaseError as e: self.logger.exception("Sphinx connection error: %s" % e) try: close_old_connections() except Exception as e: self.logger.exception("Can't reconnect: %s" % e) os.kill(os.getpid(), signal.SIGKILL) except Exception: self.logger.exception("Unhandled error in insert_data") else: return result self.logger.error("Can't insert data in 3 tries, exit process") os.kill(os.getpid(), signal.SIGKILL) def save_new_columns(self, seen): self.logger.debug("Check for new columns") for column in seen - set(self.existing): self.logger.debug("Register column %s" % column) self.index.loggercolumn_set.create(name=column) def save_column_values(self, columns): self.logger.debug("Saving values for filtered columns") ts = time.time() for column in self.filtered: values = columns.get(column) if not values: continue key = keys.KEY_COLUMN_VALUES.format(index=self.index.name, column=column) values = {v: ts for v in values} self.redis.zadd(key, **values) def prepare_query(self, name): if self.indexed: self.query = "REPLACE INTO %s (id, js, logline, %s) VALUES " % ( name, ', '.join(self.indexed)) else: self.query = "REPLACE INTO %s (id, js, logline) VALUES " % name sql_col_count = len(self.indexed) + 3 # + jd, js, logline self.values_stub = "(%s)" % ", ".join(["%s"] * sql_col_count) def load_index_columns(self): # all defined columns all_columns = list(self.index.loggercolumn_set.all()) included_columns = [c for c in all_columns if not c.excluded] filtered_columns = [c for c in included_columns if c.filtered] indexed_columns = [c for c in included_columns if c.indexed] self.existing = [c.name for c in all_columns] self.included = [c.name for c in included_columns] self.filtered = [c.name for c in filtered_columns] self.indexed = [c.name for c in indexed_columns] @staticmethod def process_js_columns(js, columns, included, seen): for key, value in list(js.items()): if key in ('pk', 'id', 'ts', 'ms', 'seq', 'model'): # escape fields reserved by Django and ALCO js['%s_x' % key] = js.pop(key) key = '%s_x' % key # save seen columns set if key not in seen: seen.add(key) if key not in included: # discard fields excluded from indexing js.pop(key) continue # save column values set if type(value) not in (bool, int, float, six.text_type): continue if value not in columns[key]: columns[key].add(value) def get_primary_keys(self, messages): """ Generate PK sequence for a list of messages.""" pkeys = [] pk = None for msg in messages: # pk is [timestamp][microseconds][randint] in 10based integer pk = int((msg['ts'] * 10**6 + msg['ms']) * 1000) + randint(0, 1000) pkeys.append(pk) self.logger.debug("first pk is %s" % pk) return pkeys