def connect_to_db() -> DatabaseHandler: """Connect to PostgreSQL.""" db_config = CommonConfig.database() retries_config = db_config.retries() assert retries_config.max_attempts() > 0, "max_tries can't be negative." db = None for attempt in range(1, retries_config.max_attempts() + 1): try: log.debug("Connecting to PostgreSQL...") db = DatabaseHandler( host=db_config.hostname(), port=db_config.port(), username=db_config.username(), password=db_config.password(), database=db_config.database_name(), ) if not db: raise ValueError("Returned value is None.") # Return the database handler upon successful connection break except Exception as ex: error_message = "Unable to connect to %(username)s@%(host)s:%(port)d/%(database)s: %(exception)s" % { 'username': db_config.username(), 'host': db_config.hostname(), 'port': db_config.port(), 'database': db_config.database_name(), 'exception': str(ex), } log.error(error_message) if attempt < retries_config.max_attempts(): log.info( f"Will retry for #{attempt} time in {retries_config.sleep_between_attempts()} seconds..." ) time.sleep(retries_config.sleep_between_attempts()) else: log.info("Out of retries, giving up and exiting...") # Don't throw any exceptions because they might be caught by # the try-catch block, and so the caller will just assume that # there was something wrong with the input data and proceed # with processing next item in the job queue (e.g. the next # story). Instead, just quit and wait for someone to restart # the whole app that requires database access. fatal_error(error_message) return db
def __init__(self, queue_name: str): """Return job broker (Celery app object) prepared for the specific queue name.""" assert queue_name, "Queue name is empty." self.__queue_name = queue_name config = CommonConfig() rabbitmq_config = config.rabbitmq() broker_uri = 'amqp://{username}:{password}@{hostname}:{port}/{vhost}'.format( username=rabbitmq_config.username(), password=rabbitmq_config.password(), hostname=rabbitmq_config.hostname(), port=rabbitmq_config.port(), vhost=rabbitmq_config.vhost(), ) db_config = CommonConfig.database() result_backend_url = 'db+postgresql+psycopg2://{username}:{password}@{hostname}:{port}/{database}'.format( username=db_config.username(), password=db_config.password(), hostname=db_config.hostname(), port=db_config.port(), database=db_config.database_name(), ) self.__app = celery.Celery(queue_name, broker=broker_uri, backend=result_backend_url) self.__app.conf.broker_connection_timeout = rabbitmq_config.timeout() # Concurrency is done by us, not Celery itself self.__app.conf.worker_concurrency = 1 self.__app.conf.broker_heartbeat = 0 # https://tech.labs.oliverwyman.com/blog/2015/04/30/making-celery-play-nice-with-rabbitmq-and-bigwig/ self.__app.conf.broker_transport_options = {'confirm_publish': True} self.__app.conf.database_table_names = { 'task': 'celery_tasks', 'group': 'celery_groups', } # Fetch only one job at a time self.__app.conf.worker_prefetch_multiplier = 1 self.__app.conf.worker_max_tasks_per_child = 1000 queue = Queue( name=queue_name, exchange=Exchange(queue_name), routing_key=queue_name, queue_arguments={ 'x-max-priority': 3, 'x-queue-mode': 'lazy', }, ) self.__app.conf.task_queues = [queue] # noinspection PyUnusedLocal def __route_task(name, args_, kwargs_, options_, task_=None, **kw_): return { 'queue': name, 'exchange': name, 'routing_key': name, } self.__app.conf.task_routes = (__route_task,)
def __init__(self, queue_name: str, rabbitmq_config: Optional[RabbitMQConfig] = None): """ Create job broker object. :param queue_name: Queue name. """ queue_name = decode_object_from_bytes_if_needed(queue_name) assert queue_name, "Queue name is empty." self.__queue_name = queue_name config = CommonConfig() if not rabbitmq_config: rabbitmq_config = config.rabbitmq() broker_uri = 'amqp://{username}:{password}@{hostname}:{port}/{vhost}'.format( username=rabbitmq_config.username(), password=rabbitmq_config.password(), hostname=rabbitmq_config.hostname(), port=rabbitmq_config.port(), vhost=rabbitmq_config.vhost(), ) db_config = CommonConfig.database() result_backend_url = 'db+postgresql+psycopg2://{username}:{password}@{hostname}:{port}/{database}'.format( username=db_config.username(), password=db_config.password(), hostname=db_config.hostname(), port=db_config.port(), database=db_config.database_name(), ) self.__app = celery.Celery(queue_name, broker=broker_uri, backend=result_backend_url) self.__app.conf.broker_connection_timeout = rabbitmq_config.timeout() # Concurrency is done by us, not Celery itself self.__app.conf.worker_concurrency = 1 self.__app.conf.broker_heartbeat = 0 # Acknowledge tasks after they get run, not before self.__app.conf.task_acks_late = 1 # https://tech.labs.oliverwyman.com/blog/2015/04/30/making-celery-play-nice-with-rabbitmq-and-bigwig/ self.__app.conf.broker_transport_options = {'confirm_publish': True} self.__app.conf.database_table_names = { 'task': 'celery_tasks', 'group': 'celery_groups', } # Fetch only one job at a time self.__app.conf.worker_prefetch_multiplier = 1 self.__app.conf.worker_max_tasks_per_child = 1000 retries_config = rabbitmq_config.retries() if retries_config: self.__app.task_publish_retry = True self.__app.task_publish_retry_policy = { 'max_retries': retries_config.max_retries(), 'interval_start': retries_config.interval_start(), 'interval_step': retries_config.interval_step(), 'interval_max': retries_config.interval_max(), } else: self.__app.task_publish_retry = False queue = Queue( name=queue_name, exchange=Exchange(queue_name), routing_key=queue_name, queue_arguments={ 'x-max-priority': 3, 'x-queue-mode': 'lazy', }, ) self.__app.conf.task_queues = [queue] # noinspection PyUnusedLocal def __route_task(name, args_, kwargs_, options_, task_=None, **kw_): return { 'queue': name, 'exchange': name, 'routing_key': name, } self.__app.conf.task_routes = (__route_task, )