def connect_to_rabbitmq(self, reconnect=False): def add_handler(queue, f, channel): logger.debug("Adding a callback to %s", queue) handler = partial(f, queue=queue) channel.basic_consume(queue, callback=handler) if self.connection and self.connection.connected and (not reconnect): return if reconnect: if self.channel is not None: self.channel.close() if self.connection is not None: self.connection.close() conn = create_amqp_connection() logger.debug("Heartbeat value: %s" % conn.heartbeat) ch = conn.channel() # Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that # we are currently using (https://www.rabbitmq.com/specification.html). # Limits are requires because consumer connection might time out when receive buffer # is full (http://stackoverflow.com/q/35438843/272770). prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count") ch.basic_qos(prefetch_size=0, prefetch_count=prefetch_count, a_global=True) add_handler("search.index", self.index_callback, ch) add_handler("search.delete", self.delete_callback, ch) self.connection = conn self.channel = ch
def _watch_impl(): def add_handler(queue, f): logger.info("Adding a callback to %s", queue) handler = partial(f, queue=queue) ch.basic_consume(queue, callback=handler) try: conn = create_amqp_connection() logger.info("Connection to RabbitMQ established") logger.debug("Heartbeat value: %s" % conn.heartbeat) ch = conn.channel() # Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that # we are currently using (https://www.rabbitmq.com/specification.html). # Limits are requires because consumer connection might time out when receive buffer # is full (http://stackoverflow.com/q/35438843/272770). prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count") ch.basic_qos(prefetch_size=0, prefetch_count=prefetch_count, a_global=True) handler = Handler() add_handler("search.index", handler.index_callback) add_handler("search.delete", handler.delete_callback) while True: logger.debug("Waiting for a message") conn.drain_events() except Exception: get_sentry().captureException() raise
def watch(args): """ Watch AMQP queues for messages. :param args: will be ignored """ try: create_amqp_connection() except socket_error as e: logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e) exit(1) try: _watch_impl() except URLError as e: logger.error("Connecting to Solr failed: %s", e) exit(1)
def _watch_impl(): def add_handler(queue, f): logger.info("Adding a callback to %s", queue) handler = partial(f, queue=queue) ch.basic_consume(queue, callback=handler) try: conn = create_amqp_connection() logger.info("Connection to RabbitMQ established") logger.debug("Heartbeat value: %s" % conn.heartbeat) ch = conn.channel() # Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that # we are currently using (https://www.rabbitmq.com/specification.html). # Limits are requires because consumer connection might time out when receive buffer # is full (http://stackoverflow.com/q/35438843/272770). prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count") ch.basic_qos(prefetch_size=0, prefetch_count=prefetch_count, a_global=True) handler = Handler() add_handler("search.index", handler.index_callback) add_handler("search.delete", handler.delete_callback) def signal_handler(signum, frame): # Setting flag to false to prevent any processes/threads blocked on # handler.queue_lock from starting `handler.process_messages` again. indexing.PROCESS_FLAG = False # Cancelling any scheduled calls try: handler.process_timer.cancel() except Exception: # In case the timer is not active it throws an exception # Simply ignore it. pass # If SIGTERM is received by the parent, make sure all its children # are killed before calling exit on the parent. children = active_children() for child in children: try: child.terminate() child.join() except Exception: pass # Finally exit exit(1) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) while True: logger.debug("Waiting for a message") conn.drain_events() except Exception: get_sentry().captureException() raise
def publish(args): """ Publish AMQP messages from the sir.message table. :param args: will be ignored """ try: create_amqp_connection() except socket_error as e: logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e) exit(1) try: _publish_impl() except Exception as e: logger.error(format_exc(e)) exit(1)
def watch(args): """ Watch AMQP queues for messages. :param [str] entity_type: Entity types to watch. """ try: create_amqp_connection() except socket_error as e: logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e) exit(1) try: entities = args["entity_type"] or SCHEMA.keys() _watch_impl(entities) except URLError as e: logger.error("Connecting to Solr failed: %s", e) exit(1)
def watch(args): """ Watch AMQP queues for messages. :param args: will be ignored """ try: create_amqp_connection() except socket_error as e: get_sentry().captureException() logger.error("Couldn't connect to RabbitMQ, check your settings") logger.error("The error was: %s", e) return try: _watch_impl() except URLError as e: get_sentry().captureException() logger.info("Connecting to Solr failed: %s", e) return
def connect_to_rabbitmq(self, reconnect=False): if self.connection and self.connection.connected and (not reconnect): return if reconnect: if self.channel is not None: self.channel.close() if self.connection is not None: self.connection.close() conn = create_amqp_connection() logger.debug("Heartbeat value: %s" % conn.heartbeat) ch = conn.channel() self.connection = conn self.channel = ch
def setup_rabbitmq(args): """ Set up the AMQP server. :param args: will be ignored """ logger.info("Connecting to RabbitMQ") conn = util.create_amqp_connection() channel = amqp.Channel(conn) channel.open() logger.info("Declaring exchanes") edecl = partial(channel.exchange_declare, auto_delete=False, durable=True) edecl("search", "direct") edecl("search.retry", "fanout") edecl("search.failed", "fanout") logger.info("Declaring queues") qdecl = partial(channel.queue_declare, auto_delete=False, durable=True) delq, _, _ = qdecl("search.delete") indq, _, _ = qdecl("search.index") retrq, _, _ = qdecl("search.retry", arguments={ "x-message-ttl": 4 * 60 * 60 * 1000, "x-dead-letter-exchange": "search"} ) failq, _, _ = qdecl("search.failed") logger.info("Binding queues") channel.queue_bind(delq, "search", "delete") channel.queue_bind(indq, "search", "index") channel.queue_bind(indq, "search", "update") channel.queue_bind(retrq, "search.retry") channel.queue_bind(failq, "search.failed") logger.info("Done") conn.close()
def setup_rabbitmq(args): """ Set up the AMQP server. :param args: will be ignored """ logger.info("Connecting to RabbitMQ") conn = util.create_amqp_connection() channel = amqp.Channel(conn) logger.info("Declaring exchanes") edecl = partial(channel.exchange_declare, auto_delete=False, durable=True) edecl("search", "direct") edecl("search.retry", "fanout") edecl("search.failed", "fanout") logger.info("Declaring queues") qdecl = partial(channel.queue_declare, auto_delete=False, durable=True) delq, _, _ = qdecl("search.delete") indq, _, _ = qdecl("search.index") retrq, _, _ = qdecl("search.retry", arguments={ "x-message-ttl": 4 * 60 * 60 * 1000, "x-dead-letter-exchange": "search"} ) failq, _, _ = qdecl("search.failed") logger.info("Binding queues") channel.queue_bind(delq, "search", "delete") channel.queue_bind(indq, "search", "index") channel.queue_bind(indq, "search", "update") channel.queue_bind(retrq, "search.retry") channel.queue_bind(failq, "search.failed") logger.info("Done") conn.close()