def send_request_to_spark_cluster(query, **params): with create_app().app_context(): message = _prepare_query_message(query, **params) rabbitmq_connection = utils.connect_to_rabbitmq( username=current_app.config['RABBITMQ_USERNAME'], password=current_app.config['RABBITMQ_PASSWORD'], host=current_app.config['RABBITMQ_HOST'], port=current_app.config['RABBITMQ_PORT'], virtual_host=current_app.config['RABBITMQ_VHOST'], error_logger=current_app.logger, ) try: channel = rabbitmq_connection.channel() channel.exchange_declare( exchange=current_app.config['SPARK_REQUEST_EXCHANGE'], exchange_type='fanout') channel.basic_publish( exchange=current_app.config['SPARK_REQUEST_EXCHANGE'], routing_key='', body=message, properties=pika.BasicProperties(delivery_mode=2, ), ) except Exception: # this is a relatively non critical part of LB for now, so just log the error and # move ahead current_app.logger.error( 'Could not send message to spark cluster: %s', ujson.dumps(message), exc_info=True)
def init_rabbitmq_connection(app): """Create a connection to the RabbitMQ server.""" global _rabbitmq if "RABBITMQ_HOST" not in app.config: app.logger.error( "RabbitMQ host:port not defined. Sleeping 2 seconds, and exiting.") sleep(2) sys.exit(-1) connection_config = { 'username': app.config['RABBITMQ_USERNAME'], 'password': app.config['RABBITMQ_PASSWORD'], 'host': app.config['RABBITMQ_HOST'], 'port': app.config['RABBITMQ_PORT'], 'virtual_host': app.config['RABBITMQ_VHOST'] } connection = utils.connect_to_rabbitmq(**connection_config, error_logger=app.logger.error, error_retry_delay=2) _rabbitmq = pika_pool.QueuedPool( create=lambda: connection, max_size=100, max_overflow=10, timeout=10, recycle=3600, stale=45, )
def connect_to_rabbitmq(self): connection_config = { 'username': current_app.config['RABBITMQ_USERNAME'], 'password': current_app.config['RABBITMQ_PASSWORD'], 'host': current_app.config['RABBITMQ_HOST'], 'port': current_app.config['RABBITMQ_PORT'], 'virtual_host': current_app.config['RABBITMQ_VHOST'], } self.connection = utils.connect_to_rabbitmq(**connection_config, error_logger=current_app.logger.error, error_retry_delay=self.ERROR_RETRY_DELAY)
def connect_to_rabbitmq(self): connection_config = { 'username': self.config.RABBITMQ_USERNAME, 'password': self.config.RABBITMQ_PASSWORD, 'host': self.config.RABBITMQ_HOST, 'port': self.config.RABBITMQ_PORT, 'virtual_host': self.config.RABBITMQ_VHOST } self.connection = utils.connect_to_rabbitmq( **connection_config, error_logger=self.log.error, error_retry_delay=self.ERROR_RETRY_DELAY)
def init_rabbitmq_connection(self): """ Initializes the connection to RabbitMQ. Note: this is a blocking function which keeps retrying if it fails to connect to RabbitMQ """ self.connection = utils.connect_to_rabbitmq( username=current_app.config['RABBITMQ_USERNAME'], password=current_app.config['RABBITMQ_PASSWORD'], host=current_app.config['RABBITMQ_HOST'], port=current_app.config['RABBITMQ_PORT'], virtual_host=current_app.config['RABBITMQ_VHOST'], error_logger=current_app.logger.error, )
def push_entities_to_queue(force=False): """ Creates a RabbitMQ connection and uses it to push entities which need their stats calculated into the queue. """ rabbitmq_connection = utils.connect_to_rabbitmq( username=config.RABBITMQ_USERNAME, password=config.RABBITMQ_PASSWORD, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, virtual_host=config.RABBITMQ_VHOST, ) channel = rabbitmq_connection.channel() channel.exchange_declare(exchange=config.BIGQUERY_EXCHANGE, exchange_type='fanout') push_users_to_queue(channel, force)
def init_rabbitmq_connection(self): """ Initializes the connection to RabbitMQ. Note: this is a blocking function which keeps retrying if it fails to connect to RabbitMQ """ self.connection = utils.connect_to_rabbitmq( username=current_app.config['RABBITMQ_USERNAME'], password=current_app.config['RABBITMQ_PASSWORD'], host=current_app.config['RABBITMQ_HOST'], port=current_app.config['RABBITMQ_PORT'], virtual_host=current_app.config['RABBITMQ_VHOST'], error_logger=current_app.logger.error, )
def push_entities_to_queue(force=False): """ Creates a RabbitMQ connection and uses it to push entities which need their stats calculated into the queue. """ rabbitmq_connection = utils.connect_to_rabbitmq( username=config.RABBITMQ_USERNAME, password=config.RABBITMQ_PASSWORD, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, virtual_host=config.RABBITMQ_VHOST, ) channel = rabbitmq_connection.channel() channel.exchange_declare(exchange=config.BIGQUERY_EXCHANGE, exchange_type='fanout') push_users_to_queue(channel, force)
def test_create_channel_to_consume(self): connection = utils.connect_to_rabbitmq( username=self.app.config['RABBITMQ_USERNAME'], password=self.app.config['RABBITMQ_PASSWORD'], host=self.app.config['RABBITMQ_HOST'], port=self.app.config['RABBITMQ_PORT'], virtual_host=self.app.config['RABBITMQ_VHOST'], error_logger=print, ) ch = utils.create_channel_to_consume( connection=connection, exchange='test', queue='test', callback_function=lambda a, b, c, d: None) self.assertIsNotNone(ch) self.assertIsInstance( ch, pika.adapters.blocking_connection.BlockingChannel)
def test_create_channel_to_consume(self): connection = utils.connect_to_rabbitmq( username=self.app.config['RABBITMQ_USERNAME'], password=self.app.config['RABBITMQ_PASSWORD'], host=self.app.config['RABBITMQ_HOST'], port=self.app.config['RABBITMQ_PORT'], virtual_host=self.app.config['RABBITMQ_VHOST'], error_logger=print, ) ch = utils.create_channel_to_consume( connection=connection, exchange='test', queue='test', callback_function=lambda a, b, c, d: None ) self.assertIsNotNone(ch) self.assertIsInstance(ch, pika.adapters.blocking_connection.BlockingChannel)