def connect_to_db() -> DatabaseHandler:
"""Connect to PostgreSQL."""
db_config = CommonConfig.database()
retries_config = db_config.retries()
assert retries_config.max_attempts() > 0, "max_tries can't be negative."
db = None
for attempt in range(1, retries_config.max_attempts() + 1):
try:
log.debug("Connecting to PostgreSQL...")
db = DatabaseHandler(
host=db_config.hostname(),
port=db_config.port(),
username=db_config.username(),
password=db_config.password(),
database=db_config.database_name(),
)
if not db:
raise ValueError("Returned value is None.")
# Return the database handler upon successful connection
break
except Exception as ex:
error_message = "Unable to connect to %(username)s@%(host)s:%(port)d/%(database)s: %(exception)s" % {
'username': db_config.username(),
'host': db_config.hostname(),
'port': db_config.port(),
'database': db_config.database_name(),
'exception': str(ex),
}
log.error(error_message)
if attempt < retries_config.max_attempts():
log.info(
f"Will retry for #{attempt} time in {retries_config.sleep_between_attempts()} seconds..."
)
time.sleep(retries_config.sleep_between_attempts())
else:
log.info("Out of retries, giving up and exiting...")
# Don't throw any exceptions because they might be caught by
# the try-catch block, and so the caller will just assume that
# there was something wrong with the input data and proceed
# with processing next item in the job queue (e.g. the next
# story). Instead, just quit and wait for someone to restart
# the whole app that requires database access.
fatal_error(error_message)
return db
def __init__(self, queue_name: str):
"""Return job broker (Celery app object) prepared for the specific queue name."""
assert queue_name, "Queue name is empty."
self.__queue_name = queue_name
config = CommonConfig()
rabbitmq_config = config.rabbitmq()
broker_uri = 'amqp://{username}:{password}@{hostname}:{port}/{vhost}'.format(
username=rabbitmq_config.username(),
password=rabbitmq_config.password(),
hostname=rabbitmq_config.hostname(),
port=rabbitmq_config.port(),
vhost=rabbitmq_config.vhost(),
)
db_config = CommonConfig.database()
result_backend_url = 'db+postgresql+psycopg2://{username}:{password}@{hostname}:{port}/{database}'.format(
username=db_config.username(),
password=db_config.password(),
hostname=db_config.hostname(),
port=db_config.port(),
database=db_config.database_name(),
)
self.__app = celery.Celery(queue_name, broker=broker_uri, backend=result_backend_url)
self.__app.conf.broker_connection_timeout = rabbitmq_config.timeout()
# Concurrency is done by us, not Celery itself
self.__app.conf.worker_concurrency = 1
self.__app.conf.broker_heartbeat = 0
# https://tech.labs.oliverwyman.com/blog/2015/04/30/making-celery-play-nice-with-rabbitmq-and-bigwig/
self.__app.conf.broker_transport_options = {'confirm_publish': True}
self.__app.conf.database_table_names = {
'task': 'celery_tasks',
'group': 'celery_groups',
}
# Fetch only one job at a time
self.__app.conf.worker_prefetch_multiplier = 1
self.__app.conf.worker_max_tasks_per_child = 1000
queue = Queue(
name=queue_name,
exchange=Exchange(queue_name),
routing_key=queue_name,
queue_arguments={
'x-max-priority': 3,
'x-queue-mode': 'lazy',
},
)
self.__app.conf.task_queues = [queue]
# noinspection PyUnusedLocal
def __route_task(name, args_, kwargs_, options_, task_=None, **kw_):
return {
'queue': name,
'exchange': name,
'routing_key': name,
}
self.__app.conf.task_routes = (__route_task,)
def __init__(self,
queue_name: str,
rabbitmq_config: Optional[RabbitMQConfig] = None):
"""
Create job broker object.
:param queue_name: Queue name.
"""
queue_name = decode_object_from_bytes_if_needed(queue_name)
assert queue_name, "Queue name is empty."
self.__queue_name = queue_name
config = CommonConfig()
if not rabbitmq_config:
rabbitmq_config = config.rabbitmq()
broker_uri = 'amqp://{username}:{password}@{hostname}:{port}/{vhost}'.format(
username=rabbitmq_config.username(),
password=rabbitmq_config.password(),
hostname=rabbitmq_config.hostname(),
port=rabbitmq_config.port(),
vhost=rabbitmq_config.vhost(),
)
db_config = CommonConfig.database()
result_backend_url = 'db+postgresql+psycopg2://{username}:{password}@{hostname}:{port}/{database}'.format(
username=db_config.username(),
password=db_config.password(),
hostname=db_config.hostname(),
port=db_config.port(),
database=db_config.database_name(),
)
self.__app = celery.Celery(queue_name,
broker=broker_uri,
backend=result_backend_url)
self.__app.conf.broker_connection_timeout = rabbitmq_config.timeout()
# Concurrency is done by us, not Celery itself
self.__app.conf.worker_concurrency = 1
self.__app.conf.broker_heartbeat = 0
# Acknowledge tasks after they get run, not before
self.__app.conf.task_acks_late = 1
# https://tech.labs.oliverwyman.com/blog/2015/04/30/making-celery-play-nice-with-rabbitmq-and-bigwig/
self.__app.conf.broker_transport_options = {'confirm_publish': True}
self.__app.conf.database_table_names = {
'task': 'celery_tasks',
'group': 'celery_groups',
}
# Fetch only one job at a time
self.__app.conf.worker_prefetch_multiplier = 1
self.__app.conf.worker_max_tasks_per_child = 1000
retries_config = rabbitmq_config.retries()
if retries_config:
self.__app.task_publish_retry = True
self.__app.task_publish_retry_policy = {
'max_retries': retries_config.max_retries(),
'interval_start': retries_config.interval_start(),
'interval_step': retries_config.interval_step(),
'interval_max': retries_config.interval_max(),
}
else:
self.__app.task_publish_retry = False
queue = Queue(
name=queue_name,
exchange=Exchange(queue_name),
routing_key=queue_name,
queue_arguments={
'x-max-priority': 3,
'x-queue-mode': 'lazy',
},
)
self.__app.conf.task_queues = [queue]
# noinspection PyUnusedLocal
def __route_task(name, args_, kwargs_, options_, task_=None, **kw_):
return {
'queue': name,
'exchange': name,
'routing_key': name,
}
self.__app.conf.task_routes = (__route_task, )
