def start(self):
""" initiates RabbitMQ connection and starts consuming from the queue
"""
with self.app.app_context():
current_app.logger.info('Spark consumer has started!')
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=current_app.config['SPARK_RESULT_EXCHANGE'],
queue=current_app.config['SPARK_RESULT_QUEUE'],
callback_function=self.callback,
auto_ack=False,
)
current_app.logger.info(
'Spark consumer attempt to start consuming!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
current_app.logger.warning(
'Spark consumer pika connection closed!')
self.connection = None
continue
self.connection.close()
def start(self):
""" initiates RabbitMQ connection and starts consuming from the queue
"""
with self.app.app_context():
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=current_app.config['SPARK_RESULT_EXCHANGE'],
queue=current_app.config['SPARK_RESULT_QUEUE'],
callback_function=self.callback,
no_ack=True,
)
self.incoming_ch.basic_qos(prefetch_count=1)
current_app.logger.info('Spark consumer started!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
current_app.logger.warning(
"Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()
def test_create_channel_to_consume(self):
connection = utils.connect_to_rabbitmq(
username=self.app.config['RABBITMQ_USERNAME'],
password=self.app.config['RABBITMQ_PASSWORD'],
host=self.app.config['RABBITMQ_HOST'],
port=self.app.config['RABBITMQ_PORT'],
virtual_host=self.app.config['RABBITMQ_VHOST'],
error_logger=print,
)
ch = utils.create_channel_to_consume(
connection=connection,
exchange='test',
queue='test',
callback_function=lambda a, b, c, d: None)
self.assertIsNotNone(ch)
self.assertIsInstance(
ch, pika.adapters.blocking_connection.BlockingChannel)
def test_create_channel_to_consume(self):
connection = utils.connect_to_rabbitmq(
username=self.app.config['RABBITMQ_USERNAME'],
password=self.app.config['RABBITMQ_PASSWORD'],
host=self.app.config['RABBITMQ_HOST'],
port=self.app.config['RABBITMQ_PORT'],
virtual_host=self.app.config['RABBITMQ_VHOST'],
error_logger=print,
)
ch = utils.create_channel_to_consume(
connection=connection,
exchange='test',
queue='test',
callback_function=lambda a, b, c, d: None
)
self.assertIsNotNone(ch)
self.assertIsInstance(ch, pika.adapters.blocking_connection.BlockingChannel)
def start(self):
""" Starts the stats calculator. This should run perpetually,
monitor the stats rabbitmq queue and calculate stats for
entities that it receives from the queue.
"""
# if no bigquery support, sleep
if not self.app.config['WRITE_TO_BIGQUERY']:
while True:
time.sleep(10000)
self.log.info('Connecting to Google BigQuery...')
stats.init_bigquery_connection()
self.log.info('Connected!')
self.log.info('Connecting to database...')
db.init_db_connection(self.app.config['SQLALCHEMY_DATABASE_URI'])
self.log.info('Connected!')
self.log.info('Connecting to redis...')
self.redis = utils.connect_to_redis(host=self.app.config['REDIS_HOST'],
port=self.app.config['REDIS_PORT'],
log=self.log.error)
self.log.info('Connected!')
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=self.app.config['STATS_EXCHANGE'],
queue=self.app.config['STATS_QUEUE'],
callback_function=self.callback,
)
self.log.info('Stats calculator started!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
self.log.info("Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()
def start(self):
""" Starts the job runner. This should run perpetually,
monitor the bigquery jobs rabbitmq queue and perform tasks for
entries in the queue.
"""
with self.app.app_context():
# if no bigquery support, sleep
if not current_app.config['WRITE_TO_BIGQUERY']:
while True:
time.sleep(10000)
current_app.logger.info('Connecting to Google BigQuery...')
self.bigquery = bigquery.create_bigquery_object()
current_app.logger.info('Connected!')
current_app.logger.info('Connecting to database...')
db.init_db_connection(current_app.config['SQLALCHEMY_DATABASE_URI'])
current_app.logger.info('Connected!')
current_app.logger.info('Connecting to redis...')
self.redis = utils.connect_to_redis(host=current_app.config['REDIS_HOST'], port=current_app.config['REDIS_PORT'], log=current_app.logger.error)
current_app.logger.info('Connected!')
while True:
self.init_rabbitmq_connection()
self.incoming_ch = utils.create_channel_to_consume(
connection=self.connection,
exchange=current_app.config['BIGQUERY_EXCHANGE'],
queue=current_app.config['BIGQUERY_QUEUE'],
callback_function=self.callback,
)
current_app.logger.info('Stats calculator started!')
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
current_app.logger.warning("Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()