def send_request_to_spark_cluster(query, **params):
with create_app().app_context():
message = _prepare_query_message(query, **params)
rabbitmq_connection = utils.connect_to_rabbitmq(
username=current_app.config['RABBITMQ_USERNAME'],
password=current_app.config['RABBITMQ_PASSWORD'],
host=current_app.config['RABBITMQ_HOST'],
port=current_app.config['RABBITMQ_PORT'],
virtual_host=current_app.config['RABBITMQ_VHOST'],
error_logger=current_app.logger,
)
try:
channel = rabbitmq_connection.channel()
channel.exchange_declare(
exchange=current_app.config['SPARK_REQUEST_EXCHANGE'],
exchange_type='fanout')
channel.basic_publish(
exchange=current_app.config['SPARK_REQUEST_EXCHANGE'],
routing_key='',
body=message,
properties=pika.BasicProperties(delivery_mode=2, ),
)
except Exception:
# this is a relatively non critical part of LB for now, so just log the error and
# move ahead
current_app.logger.error(
'Could not send message to spark cluster: %s',
ujson.dumps(message),
exc_info=True)
def init_rabbitmq_connection(app):
"""Create a connection to the RabbitMQ server."""
global _rabbitmq
if "RABBITMQ_HOST" not in app.config:
app.logger.error(
"RabbitMQ host:port not defined. Sleeping 2 seconds, and exiting.")
sleep(2)
sys.exit(-1)
connection_config = {
'username': app.config['RABBITMQ_USERNAME'],
'password': app.config['RABBITMQ_PASSWORD'],
'host': app.config['RABBITMQ_HOST'],
'port': app.config['RABBITMQ_PORT'],
'virtual_host': app.config['RABBITMQ_VHOST']
}
connection = utils.connect_to_rabbitmq(**connection_config,
error_logger=app.logger.error,
error_retry_delay=2)
_rabbitmq = pika_pool.QueuedPool(
create=lambda: connection,
max_size=100,
max_overflow=10,
timeout=10,
recycle=3600,
stale=45,
)
def connect_to_rabbitmq(self):
connection_config = {
'username': current_app.config['RABBITMQ_USERNAME'],
'password': current_app.config['RABBITMQ_PASSWORD'],
'host': current_app.config['RABBITMQ_HOST'],
'port': current_app.config['RABBITMQ_PORT'],
'virtual_host': current_app.config['RABBITMQ_VHOST'],
}
self.connection = utils.connect_to_rabbitmq(**connection_config,
error_logger=current_app.logger.error,
error_retry_delay=self.ERROR_RETRY_DELAY)
def connect_to_rabbitmq(self):
connection_config = {
'username': self.config.RABBITMQ_USERNAME,
'password': self.config.RABBITMQ_PASSWORD,
'host': self.config.RABBITMQ_HOST,
'port': self.config.RABBITMQ_PORT,
'virtual_host': self.config.RABBITMQ_VHOST
}
self.connection = utils.connect_to_rabbitmq(
**connection_config,
error_logger=self.log.error,
error_retry_delay=self.ERROR_RETRY_DELAY)
def init_rabbitmq_connection(self):
""" Initializes the connection to RabbitMQ.
Note: this is a blocking function which keeps retrying if it fails
to connect to RabbitMQ
"""
self.connection = utils.connect_to_rabbitmq(
username=current_app.config['RABBITMQ_USERNAME'],
password=current_app.config['RABBITMQ_PASSWORD'],
host=current_app.config['RABBITMQ_HOST'],
port=current_app.config['RABBITMQ_PORT'],
virtual_host=current_app.config['RABBITMQ_VHOST'],
error_logger=current_app.logger.error,
)
def push_entities_to_queue(force=False):
""" Creates a RabbitMQ connection and uses it to push entities which
need their stats calculated into the queue.
"""
rabbitmq_connection = utils.connect_to_rabbitmq(
username=config.RABBITMQ_USERNAME,
password=config.RABBITMQ_PASSWORD,
host=config.RABBITMQ_HOST,
port=config.RABBITMQ_PORT,
virtual_host=config.RABBITMQ_VHOST,
)
channel = rabbitmq_connection.channel()
channel.exchange_declare(exchange=config.BIGQUERY_EXCHANGE, exchange_type='fanout')
push_users_to_queue(channel, force)
def init_rabbitmq_connection(self):
""" Initializes the connection to RabbitMQ.
Note: this is a blocking function which keeps retrying if it fails
to connect to RabbitMQ
"""
self.connection = utils.connect_to_rabbitmq(
username=current_app.config['RABBITMQ_USERNAME'],
password=current_app.config['RABBITMQ_PASSWORD'],
host=current_app.config['RABBITMQ_HOST'],
port=current_app.config['RABBITMQ_PORT'],
virtual_host=current_app.config['RABBITMQ_VHOST'],
error_logger=current_app.logger.error,
)
def push_entities_to_queue(force=False):
""" Creates a RabbitMQ connection and uses it to push entities which
need their stats calculated into the queue.
"""
rabbitmq_connection = utils.connect_to_rabbitmq(
username=config.RABBITMQ_USERNAME,
password=config.RABBITMQ_PASSWORD,
host=config.RABBITMQ_HOST,
port=config.RABBITMQ_PORT,
virtual_host=config.RABBITMQ_VHOST,
)
channel = rabbitmq_connection.channel()
channel.exchange_declare(exchange=config.BIGQUERY_EXCHANGE,
exchange_type='fanout')
push_users_to_queue(channel, force)
def test_create_channel_to_consume(self):
connection = utils.connect_to_rabbitmq(
username=self.app.config['RABBITMQ_USERNAME'],
password=self.app.config['RABBITMQ_PASSWORD'],
host=self.app.config['RABBITMQ_HOST'],
port=self.app.config['RABBITMQ_PORT'],
virtual_host=self.app.config['RABBITMQ_VHOST'],
error_logger=print,
)
ch = utils.create_channel_to_consume(
connection=connection,
exchange='test',
queue='test',
callback_function=lambda a, b, c, d: None)
self.assertIsNotNone(ch)
self.assertIsInstance(
ch, pika.adapters.blocking_connection.BlockingChannel)
def test_create_channel_to_consume(self):
connection = utils.connect_to_rabbitmq(
username=self.app.config['RABBITMQ_USERNAME'],
password=self.app.config['RABBITMQ_PASSWORD'],
host=self.app.config['RABBITMQ_HOST'],
port=self.app.config['RABBITMQ_PORT'],
virtual_host=self.app.config['RABBITMQ_VHOST'],
error_logger=print,
)
ch = utils.create_channel_to_consume(
connection=connection,
exchange='test',
queue='test',
callback_function=lambda a, b, c, d: None
)
self.assertIsNotNone(ch)
self.assertIsInstance(ch, pika.adapters.blocking_connection.BlockingChannel)
