def start(self): """ initiates RabbitMQ connection and starts consuming from the queue """ with self.app.app_context(): current_app.logger.info('Spark consumer has started!') while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=current_app.config['SPARK_RESULT_EXCHANGE'], queue=current_app.config['SPARK_RESULT_QUEUE'], callback_function=self.callback, auto_ack=False, ) current_app.logger.info( 'Spark consumer attempt to start consuming!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: current_app.logger.warning( 'Spark consumer pika connection closed!') self.connection = None continue self.connection.close()
def start(self): """ initiates RabbitMQ connection and starts consuming from the queue """ with self.app.app_context(): while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=current_app.config['SPARK_RESULT_EXCHANGE'], queue=current_app.config['SPARK_RESULT_QUEUE'], callback_function=self.callback, no_ack=True, ) self.incoming_ch.basic_qos(prefetch_count=1) current_app.logger.info('Spark consumer started!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: current_app.logger.warning( "Connection to rabbitmq closed. Re-opening.") self.connection = None continue self.connection.close()
def test_create_channel_to_consume(self): connection = utils.connect_to_rabbitmq( username=self.app.config['RABBITMQ_USERNAME'], password=self.app.config['RABBITMQ_PASSWORD'], host=self.app.config['RABBITMQ_HOST'], port=self.app.config['RABBITMQ_PORT'], virtual_host=self.app.config['RABBITMQ_VHOST'], error_logger=print, ) ch = utils.create_channel_to_consume( connection=connection, exchange='test', queue='test', callback_function=lambda a, b, c, d: None) self.assertIsNotNone(ch) self.assertIsInstance( ch, pika.adapters.blocking_connection.BlockingChannel)
def test_create_channel_to_consume(self): connection = utils.connect_to_rabbitmq( username=self.app.config['RABBITMQ_USERNAME'], password=self.app.config['RABBITMQ_PASSWORD'], host=self.app.config['RABBITMQ_HOST'], port=self.app.config['RABBITMQ_PORT'], virtual_host=self.app.config['RABBITMQ_VHOST'], error_logger=print, ) ch = utils.create_channel_to_consume( connection=connection, exchange='test', queue='test', callback_function=lambda a, b, c, d: None ) self.assertIsNotNone(ch) self.assertIsInstance(ch, pika.adapters.blocking_connection.BlockingChannel)
def start(self): """ Starts the stats calculator. This should run perpetually, monitor the stats rabbitmq queue and calculate stats for entities that it receives from the queue. """ # if no bigquery support, sleep if not self.app.config['WRITE_TO_BIGQUERY']: while True: time.sleep(10000) self.log.info('Connecting to Google BigQuery...') stats.init_bigquery_connection() self.log.info('Connected!') self.log.info('Connecting to database...') db.init_db_connection(self.app.config['SQLALCHEMY_DATABASE_URI']) self.log.info('Connected!') self.log.info('Connecting to redis...') self.redis = utils.connect_to_redis(host=self.app.config['REDIS_HOST'], port=self.app.config['REDIS_PORT'], log=self.log.error) self.log.info('Connected!') while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=self.app.config['STATS_EXCHANGE'], queue=self.app.config['STATS_QUEUE'], callback_function=self.callback, ) self.log.info('Stats calculator started!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: self.log.info("Connection to rabbitmq closed. Re-opening.") self.connection = None continue self.connection.close()
def start(self): """ Starts the job runner. This should run perpetually, monitor the bigquery jobs rabbitmq queue and perform tasks for entries in the queue. """ with self.app.app_context(): # if no bigquery support, sleep if not current_app.config['WRITE_TO_BIGQUERY']: while True: time.sleep(10000) current_app.logger.info('Connecting to Google BigQuery...') self.bigquery = bigquery.create_bigquery_object() current_app.logger.info('Connected!') current_app.logger.info('Connecting to database...') db.init_db_connection(current_app.config['SQLALCHEMY_DATABASE_URI']) current_app.logger.info('Connected!') current_app.logger.info('Connecting to redis...') self.redis = utils.connect_to_redis(host=current_app.config['REDIS_HOST'], port=current_app.config['REDIS_PORT'], log=current_app.logger.error) current_app.logger.info('Connected!') while True: self.init_rabbitmq_connection() self.incoming_ch = utils.create_channel_to_consume( connection=self.connection, exchange=current_app.config['BIGQUERY_EXCHANGE'], queue=current_app.config['BIGQUERY_QUEUE'], callback_function=self.callback, ) current_app.logger.info('Stats calculator started!') try: self.incoming_ch.start_consuming() except pika.exceptions.ConnectionClosed: current_app.logger.warning("Connection to rabbitmq closed. Re-opening.") self.connection = None continue self.connection.close()