def connect_to_rabbitmq(self, reconnect=False):
def add_handler(queue, f, channel):
logger.debug("Adding a callback to %s", queue)
handler = partial(f, queue=queue)
channel.basic_consume(queue, callback=handler)
if self.connection and self.connection.connected and (not reconnect):
return
if reconnect:
if self.channel is not None:
self.channel.close()
if self.connection is not None:
self.connection.close()
conn = create_amqp_connection()
logger.debug("Heartbeat value: %s" % conn.heartbeat)
ch = conn.channel()
# Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that
# we are currently using (https://www.rabbitmq.com/specification.html).
# Limits are requires because consumer connection might time out when receive buffer
# is full (http://stackoverflow.com/q/35438843/272770).
prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count")
ch.basic_qos(prefetch_size=0,
prefetch_count=prefetch_count,
a_global=True)
add_handler("search.index", self.index_callback, ch)
add_handler("search.delete", self.delete_callback, ch)
self.connection = conn
self.channel = ch
def connect_to_rabbitmq(self, reconnect=False):
def add_handler(queue, f, channel):
logger.debug("Adding a callback to %s", queue)
handler = partial(f, queue=queue)
channel.basic_consume(queue, callback=handler)
if self.connection and self.connection.connected and (not reconnect):
return
if reconnect:
if self.channel is not None:
self.channel.close()
if self.connection is not None:
self.connection.close()
conn = create_amqp_connection()
logger.debug("Heartbeat value: %s" % conn.heartbeat)
ch = conn.channel()
# Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that
# we are currently using (https://www.rabbitmq.com/specification.html).
# Limits are requires because consumer connection might time out when receive buffer
# is full (http://stackoverflow.com/q/35438843/272770).
prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count")
ch.basic_qos(prefetch_size=0, prefetch_count=prefetch_count, a_global=True)
add_handler("search.index", self.index_callback, ch)
add_handler("search.delete", self.delete_callback, ch)
self.connection = conn
self.channel = ch
def _watch_impl():
def add_handler(queue, f):
logger.info("Adding a callback to %s", queue)
handler = partial(f, queue=queue)
ch.basic_consume(queue, callback=handler)
try:
conn = create_amqp_connection()
logger.info("Connection to RabbitMQ established")
logger.debug("Heartbeat value: %s" % conn.heartbeat)
ch = conn.channel()
# Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that
# we are currently using (https://www.rabbitmq.com/specification.html).
# Limits are requires because consumer connection might time out when receive buffer
# is full (http://stackoverflow.com/q/35438843/272770).
prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count")
ch.basic_qos(prefetch_size=0,
prefetch_count=prefetch_count,
a_global=True)
handler = Handler()
add_handler("search.index", handler.index_callback)
add_handler("search.delete", handler.delete_callback)
while True:
logger.debug("Waiting for a message")
conn.drain_events()
except Exception:
get_sentry().captureException()
raise
def watch(args):
"""
Watch AMQP queues for messages.
:param args: will be ignored
"""
try:
create_amqp_connection()
except socket_error as e:
logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e)
exit(1)
try:
_watch_impl()
except URLError as e:
logger.error("Connecting to Solr failed: %s", e)
exit(1)
def watch(args):
"""
Watch AMQP queues for messages.
:param args: will be ignored
"""
try:
create_amqp_connection()
except socket_error as e:
logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e)
exit(1)
try:
_watch_impl()
except URLError as e:
logger.error("Connecting to Solr failed: %s", e)
exit(1)
def _watch_impl():
def add_handler(queue, f):
logger.info("Adding a callback to %s", queue)
handler = partial(f, queue=queue)
ch.basic_consume(queue, callback=handler)
try:
conn = create_amqp_connection()
logger.info("Connection to RabbitMQ established")
logger.debug("Heartbeat value: %s" % conn.heartbeat)
ch = conn.channel()
# Keep in mind that `prefetch_size` is not supported by the version of RabbitMQ that
# we are currently using (https://www.rabbitmq.com/specification.html).
# Limits are requires because consumer connection might time out when receive buffer
# is full (http://stackoverflow.com/q/35438843/272770).
prefetch_count = config.CFG.getint("rabbitmq", "prefetch_count")
ch.basic_qos(prefetch_size=0,
prefetch_count=prefetch_count,
a_global=True)
handler = Handler()
add_handler("search.index", handler.index_callback)
add_handler("search.delete", handler.delete_callback)
def signal_handler(signum, frame):
# Setting flag to false to prevent any processes/threads blocked on
# handler.queue_lock from starting `handler.process_messages` again.
indexing.PROCESS_FLAG = False
# Cancelling any scheduled calls
try:
handler.process_timer.cancel()
except Exception:
# In case the timer is not active it throws an exception
# Simply ignore it.
pass
# If SIGTERM is received by the parent, make sure all its children
# are killed before calling exit on the parent.
children = active_children()
for child in children:
try:
child.terminate()
child.join()
except Exception:
pass
# Finally exit
exit(1)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
while True:
logger.debug("Waiting for a message")
conn.drain_events()
except Exception:
get_sentry().captureException()
raise
def publish(args):
"""
Publish AMQP messages from the sir.message table.
:param args: will be ignored
"""
try:
create_amqp_connection()
except socket_error as e:
logger.error("Couldn't connect to RabbitMQ, check your settings. %s",
e)
exit(1)
try:
_publish_impl()
except Exception as e:
logger.error(format_exc(e))
exit(1)
def watch(args):
"""
Watch AMQP queues for messages.
:param [str] entity_type: Entity types to watch.
"""
try:
create_amqp_connection()
except socket_error as e:
logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e)
exit(1)
try:
entities = args["entity_type"] or SCHEMA.keys()
_watch_impl(entities)
except URLError as e:
logger.error("Connecting to Solr failed: %s", e)
exit(1)
def watch(args):
"""
Watch AMQP queues for messages.
:param args: will be ignored
"""
try:
create_amqp_connection()
except socket_error as e:
get_sentry().captureException()
logger.error("Couldn't connect to RabbitMQ, check your settings")
logger.error("The error was: %s", e)
return
try:
_watch_impl()
except URLError as e:
get_sentry().captureException()
logger.info("Connecting to Solr failed: %s", e)
return
def connect_to_rabbitmq(self, reconnect=False):
if self.connection and self.connection.connected and (not reconnect):
return
if reconnect:
if self.channel is not None:
self.channel.close()
if self.connection is not None:
self.connection.close()
conn = create_amqp_connection()
logger.debug("Heartbeat value: %s" % conn.heartbeat)
ch = conn.channel()
self.connection = conn
self.channel = ch
def setup_rabbitmq(args):
"""
Set up the AMQP server.
:param args: will be ignored
"""
logger.info("Connecting to RabbitMQ")
conn = util.create_amqp_connection()
channel = amqp.Channel(conn)
channel.open()
logger.info("Declaring exchanes")
edecl = partial(channel.exchange_declare,
auto_delete=False,
durable=True)
edecl("search", "direct")
edecl("search.retry", "fanout")
edecl("search.failed", "fanout")
logger.info("Declaring queues")
qdecl = partial(channel.queue_declare,
auto_delete=False,
durable=True)
delq, _, _ = qdecl("search.delete")
indq, _, _ = qdecl("search.index")
retrq, _, _ = qdecl("search.retry", arguments={
"x-message-ttl":
4 * 60 * 60 * 1000,
"x-dead-letter-exchange":
"search"}
)
failq, _, _ = qdecl("search.failed")
logger.info("Binding queues")
channel.queue_bind(delq, "search", "delete")
channel.queue_bind(indq, "search", "index")
channel.queue_bind(indq, "search", "update")
channel.queue_bind(retrq, "search.retry")
channel.queue_bind(failq, "search.failed")
logger.info("Done")
conn.close()
def setup_rabbitmq(args):
"""
Set up the AMQP server.
:param args: will be ignored
"""
logger.info("Connecting to RabbitMQ")
conn = util.create_amqp_connection()
channel = amqp.Channel(conn)
logger.info("Declaring exchanes")
edecl = partial(channel.exchange_declare,
auto_delete=False,
durable=True)
edecl("search", "direct")
edecl("search.retry", "fanout")
edecl("search.failed", "fanout")
logger.info("Declaring queues")
qdecl = partial(channel.queue_declare,
auto_delete=False,
durable=True)
delq, _, _ = qdecl("search.delete")
indq, _, _ = qdecl("search.index")
retrq, _, _ = qdecl("search.retry", arguments={
"x-message-ttl":
4 * 60 * 60 * 1000,
"x-dead-letter-exchange":
"search"}
)
failq, _, _ = qdecl("search.failed")
logger.info("Binding queues")
channel.queue_bind(delq, "search", "delete")
channel.queue_bind(indq, "search", "index")
channel.queue_bind(indq, "search", "update")
channel.queue_bind(retrq, "search.retry")
channel.queue_bind(failq, "search.failed")
logger.info("Done")
conn.close()
