def get_bindings(self, exchange: str) -> Optional[List[Dict[str, str]]]:
if not self.exchange_exists(exchange):
log.critical("Does not exist")
return None
host = self.variables.get("host", "")
schema = ""
if not host.startswith("http"):
if Env.to_bool(self.variables.get("ssl_enabled")):
schema = "https://"
else:
schema = "http://"
port = self.variables.get("management_port")
# url-encode unsafe characters by also including / (thanks to safe parameter)
# / -> %2F
vhost = urllib.parse.quote(self.variables.get("vhost", "/"), safe="")
user = self.variables.get("user")
password = self.variables.get("password")
# API Reference:
# A list of all bindings in which a given exchange is the source.
r = requests.get(
f"{schema}{host}:{port}/api/exchanges/{vhost}/{exchange}/bindings/source",
auth=HTTPBasicAuth(user, password),
verify=False,
)
response = r.json()
if r.status_code != 200: # pragma: no cover
raise RestApiException(
{"RabbitMQ": response.get("error", "Unknown error")},
status_code=r.status_code,
)
bindings = []
for row in response:
# row == {
# 'source': exchange-name,
# 'vhost': probably '/',
# 'destination': queue-or-dest-exchange-name,
# 'destination_type': 'queue' or 'exchange',
# 'routing_key': routing_key,
# 'arguments': Dict,
# 'properties_key': ?? as routing_key?
# }
bindings.append({
"exchange": row["source"],
"routing_key": row["routing_key"],
"queue": row["destination"],
})
return bindings
def connect(self, **kwargs):
variables = self.variables.copy()
# Beware, if you specify a user different by the default,
# then the send method will fail to to PRECONDITION_FAILED because
# the user_id will not pass the verification
# Locally save self.variables + kwargs to be used in send()
variables.update(kwargs)
ssl_enabled = Env.to_bool(variables.get("ssl_enabled"))
log.info("Connecting to the Rabbit (SSL = {})", ssl_enabled)
if (host := variables.get("host")) is None:
raise ServiceUnavailable("Missing hostname")
variables = self.variables.copy()
variables.update(kwargs)
if (host := variables.get("host")) is None: # pragma: no cover
raise ServiceUnavailable("Missing hostname")
if (user := variables.get("user")) is None: # pragma: no cover
raise ServiceUnavailable("Missing credentials")
if (password := variables.get("password")) is None: # pragma: no cover
raise ServiceUnavailable("Missing credentials")
port = Env.get_int(variables.get("port"), 21)
ssl_enabled = Env.to_bool(variables.get("ssl_enabled"))
if ssl_enabled:
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.load_default_certs()
# Disable certificate verification:
# context.verify_mode = ssl.CERT_NONE
# Enable certificate verification:
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = False
# Path to pem file to verify self signed certificates
context.load_verify_locations(cafile=SSL_CERTIFICATE)
# System CA to verify true cerificates
context.load_verify_locations(cafile=certifi.where())
def test_env(self, faker: Faker) -> None:
assert not Env.to_bool(None)
assert Env.to_bool(None, True)
assert not Env.to_bool(False)
assert Env.to_bool(True)
assert not Env.to_bool(0)
assert Env.to_bool(1)
assert Env.to_bool(1 + faker.pyint())
assert Env.to_bool(-faker.pyint() - 1)
assert not Env.to_bool("")
assert not Env.to_bool("false")
assert not Env.to_bool("False")
assert not Env.to_bool("FALSE")
assert Env.to_bool("true")
assert Env.to_bool("True")
assert Env.to_bool("TRUE")
assert Env.to_bool(faker.pystr())
assert not Env.to_bool(object)
assert Env.to_bool(object, True)
random_default = faker.pyint()
assert Env.to_int(None) == 0
assert Env.to_int(None, random_default) == random_default
assert Env.to_int(random_default) == random_default
assert Env.to_int("42") == 42
assert Env.to_int("-42") == -42
assert Env.to_int(str(random_default)) == random_default
assert Env.to_int(faker.pystr()) == 0
assert Env.to_int(faker.pystr(), random_default) == random_default
assert Env.to_bool(object) == 0
prefix = faker.pystr().lower()
var1 = faker.pystr()
var2 = faker.pystr().lower()
var3 = faker.pystr().upper()
val1 = faker.pystr()
val2 = faker.pystr()
val3 = faker.pystr()
os.environ[f"{prefix}_{var1}"] = val1
os.environ[f"{prefix}_{var2}"] = val2
os.environ[f"{prefix}_{var3}"] = val3
variables = Env.load_variables_group(prefix)
assert variables is not None
assert isinstance(variables, dict)
assert len(variables) == 3
assert var2 in variables
assert var3 not in variables
assert var3.lower() in variables
assert variables.get(var1.lower()) == val1
assert variables.get(var2.lower()) == val2
assert variables.get(var3.lower()) == val3
def load_connectors(cls, path: Path, module: str, services: Services) -> Services:
main_folder = path.joinpath(CONNECTORS_FOLDER)
if not main_folder.is_dir():
log.debug("Connectors folder not found: {}", main_folder)
return services
for connector in main_folder.iterdir():
if not connector.is_dir():
continue
connector_name = connector.name
if connector_name.startswith("_"):
continue
# This is the only exception... we should rename sqlalchemy as alchemy
if connector_name == "sqlalchemy":
variables = Env.load_variables_group(prefix="alchemy")
else:
variables = Env.load_variables_group(prefix=connector_name)
if not Env.to_bool(
variables.get("enable_connector", True)
): # pragma: no cover
log.debug("{} connector is disabled", connector_name)
continue
external = False
if "host" in variables:
if host := variables.get("host"):
external = cls.is_external(host)
# HOST found in variables but empty... never happens during tests
else: # pragma: no cover
variables["enable"] = "0"
enabled = Env.to_bool(variables.get("enable"))
# Celery is always enabled, if connector is enabled
# No further check is needed on host/external
available = enabled or external or connector_name == "celery"
if not available:
continue
connector_module = Connector.get_module(connector_name, module)
connector_class = Connector.get_class(connector_module)
# Can't test connector misconfiguration...
if not connector_class: # pragma: no cover
log.error("No connector class found in {}/{}", main_folder, connector)
continue
try:
# This is to test the Connector compliance,
# i.e. to verify instance and get_instance in the connector module
# and verify that the Connector can be instanced
connector_module.instance
connector_module.get_instance
connector_class()
except AttributeError as e: # pragma: no cover
print_and_exit(e)
services[connector_name] = variables
log.debug("Got class definition for {}", connector_class)
def load_connectors(cls, path: str, module: str, services: Services) -> Services:
main_folder = os.path.join(path, CONNECTORS_FOLDER)
if not os.path.isdir(main_folder):
log.debug("Connectors folder not found: {}", main_folder)
return services
for connector in os.listdir(main_folder):
connector_path = os.path.join(path, CONNECTORS_FOLDER, connector)
if not os.path.isdir(connector_path):
continue
if connector.startswith("_"):
continue
# This is the only exception... we should rename sqlalchemy as alchemy
if connector == "sqlalchemy":
variables = Env.load_variables_group(prefix="alchemy")
else:
variables = Env.load_variables_group(prefix=connector)
if not Env.to_bool(
variables.get("enable_connector", True)
): # pragma: no cover
log.info("{} connector is disabled", connector)
continue
# if host is not in variables (like for Celery) do not consider it
external = False
if "host" in variables:
if host := variables.get("host"):
external = cls.is_external(host)
# HOST found in variables but empty... never happens during tests
else: # pragma: no cover
variables["enable"] = "0"
enabled = Env.to_bool(variables.get("enable"))
available = enabled or external
if not available:
continue
connector_module = Connector.get_module(connector, module)
connector_class = Connector.get_class(connector_module)
# Can't test connector misconfiguration...
if not connector_class: # pragma: no cover
log.error("No connector class found in {}/{}", main_folder, connector)
continue
try:
# This is to test the Connector compliance,
# i.e. to verify instance and get_instance in the connector module
# and verify that the Connector can be instanced
connector_module.instance
connector_module.get_instance
connector_class()
except AttributeError as e: # pragma: no cover
print_and_exit(e)
services[connector] = variables
log.debug("Got class definition for {}", connector_class)
def connect(self, **kwargs):
variables = self.variables.copy()
variables.update(kwargs)
broker = variables.get("broker")
if broker is None: # pragma: no cover
print_and_exit("Unable to start Celery, missing broker service")
if broker == "RABBIT":
service_vars = Env.load_variables_group(prefix="rabbitmq")
self.celery_app.conf.broker_use_ssl = Env.to_bool(
service_vars.get("ssl_enabled"))
self.celery_app.conf.broker_url = self.get_rabbit_url(
service_vars, protocol="amqp")
elif broker == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
self.celery_app.conf.broker_use_ssl = False
self.celery_app.conf.broker_url = self.get_redis_url(
service_vars, protocol="redis")
else: # pragma: no cover
print_and_exit(
"Unable to start Celery: unknown broker service: {}", broker)
log.info(
"Configured {} as broker {}",
broker,
obfuscate_url(self.celery_app.conf.broker_url),
)
# From the guide: "Default: Taken from broker_url."
# But it is not true, connection fails if not explicitly set
self.celery_app.conf.broker_read_url = self.celery_app.conf.broker_url
self.celery_app.conf.broker_write_url = self.celery_app.conf.broker_url
backend = variables.get("backend", broker)
if backend == "RABBIT":
service_vars = Env.load_variables_group(prefix="rabbitmq")
log.warning(
"RABBIT backend is quite limited and not fully supported. "
"Consider to enable Redis or MongoDB as a backend database")
self.celery_app.conf.result_backend = self.get_rabbit_url(
service_vars, protocol="rpc")
elif backend == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
self.celery_app.conf.result_backend = self.get_redis_url(
service_vars, protocol="redis")
# set('redis_backend_use_ssl', kwargs.get('redis_backend_use_ssl'))
elif backend == "MONGODB":
service_vars = Env.load_variables_group(prefix="mongo")
self.celery_app.conf.result_backend = self.get_mongodb_url(
service_vars, protocol="mongodb")
else: # pragma: no cover
print_and_exit(
"Unable to start Celery: unknown backend service: {}", backend)
log.info(
"Configured {} as backend {}",
backend,
obfuscate_url(self.celery_app.conf.result_backend),
)
# Should be enabled?
# Default: Disabled by default (transient messages).
# If set to True, result messages will be persistent.
# This means the messages won’t be lost after a broker restart.
# self.celery_app.conf.result_persistent = True
# Skip initial warnings, avoiding pickle format (deprecated)
self.celery_app.conf.accept_content = ["json"]
self.celery_app.conf.task_serializer = "json"
self.celery_app.conf.result_serializer = "json"
# Already enabled by default to use UTC
# self.celery_app.conf.enable_utc
# self.celery_app.conf.timezone
# Not needed, because tasks are dynamcally injected
# self.celery_app.conf.imports
# self.celery_app.conf.includes
# Max priority default value for all queues
# Required to be able to set priority parameter on task calls
self.celery_app.conf.task_queue_max_priority = 10
# Default priority for taks (if not specified)
self.celery_app.conf.task_default_priority = 5
# If you want to apply a more strict priority to items
# probably prefetching should also be disabled:
# Late ack means the task messages will be acknowledged after the task
# has been executed, not just before (the default behavior).
# self.celery_app.conf.task_acks_late = True
# How many messages to prefetch at a time multiplied by the number
# of concurrent processes. The default is 4 (four messages for each process).
# The default setting is usually a good choice, however – if you have very
# long running tasks waiting in the queue and you have to start the workers,
# note that the first worker to start will receive four times the number
# of messages initially. Thus the tasks may not be fairly distributed to
# the workers. To disable prefetching, set worker_prefetch_multiplier to 1.
# Changing that setting to 0 will allow the worker to keep consuming as many
# messages as it wants.
self.celery_app.conf.worker_prefetch_multiplier = 1
if Env.get_bool("CELERYBEAT_ENABLED"):
CeleryExt.CELERYBEAT_SCHEDULER = backend
if backend == "MONGODB":
service_vars = Env.load_variables_group(prefix="mongo")
url = self.get_mongodb_url(service_vars, protocol="mongodb")
SCHEDULER_DB = "celery"
self.celery_app.conf[
"CELERY_MONGODB_SCHEDULER_DB"] = SCHEDULER_DB
self.celery_app.conf[
"CELERY_MONGODB_SCHEDULER_COLLECTION"] = "schedules"
self.celery_app.conf["CELERY_MONGODB_SCHEDULER_URL"] = url
import mongoengine
m = mongoengine.connect(SCHEDULER_DB, host=url)
log.info("Celery-beat connected to MongoDB: {}", m)
elif backend == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
url = self.get_redis_url(service_vars, protocol="redis")
self.celery_app.conf["REDBEAT_REDIS_URL"] = url
self.celery_app.conf["REDBEAT_KEY_PREFIX"] = REDBEAT_KEY_PREFIX
log.info("Celery-beat connected to Redis: {}",
obfuscate_url(url))
else: # pragma: no cover
log.warning(
"Cannot configure celery beat scheduler with backend: {}",
backend)
# self.disconnected = False
conf = self.celery_app.conf
# Replace the previous App with new settings
self.celery_app = Celery("RAPyDo",
broker=conf["broker_url"],
backend=conf["result_backend"])
self.celery_app.conf = conf
for funct in Meta.get_celery_tasks(f"{CUSTOM_PACKAGE}.tasks"):
# Weird errors due to celery-stubs?
# "Callable[[], Any]" has no attribute "register"
# The code is correct... let's ignore it
self.celery_app.tasks.register(funct) # type: ignore
return self
def connect(self, **kwargs: str) -> "CeleryExt":
variables = self.variables.copy()
variables.update(kwargs)
broker = variables.get("broker_service")
if HOST_TYPE == DOCS: # pragma: no cover
broker = "RABBIT"
if broker is None: # pragma: no cover
print_and_exit("Unable to start Celery, missing broker service")
if broker == "RABBIT":
service_vars = Env.load_variables_group(prefix="rabbitmq")
if Env.to_bool(service_vars.get("ssl_enabled")):
# The setting can be a dict with the following keys:
# ssl_cert_reqs (required): one of the SSLContext.verify_mode values:
# ssl.CERT_NONE
# ssl.CERT_OPTIONAL
# ssl.CERT_REQUIRED
# ssl_ca_certs (optional): path to the CA certificate
# ssl_certfile (optional): path to the client certificate
# ssl_keyfile (optional): path to the client key
server_hostname = RabbitExt.get_hostname(
service_vars.get("host", ""))
force_self_signed = Env.get_bool("SSL_FORCE_SELF_SIGNED")
ca_certs = (SSL_CERTIFICATE if server_hostname == "localhost"
or force_self_signed else certifi.where())
self.celery_app.conf.broker_use_ssl = {
# 'keyfile': '/var/ssl/private/worker-key.pem',
# 'certfile': '/var/ssl/amqp-server-cert.pem',
# 'ca_certs': '/var/ssl/myca.pem',
# 'cert_reqs': ssl.CERT_REQUIRED
# 'cert_reqs': ssl.CERT_OPTIONAL
"cert_reqs": ssl.CERT_REQUIRED,
"server_hostname": server_hostname,
"ca_certs": ca_certs,
}
self.celery_app.conf.broker_url = self.get_rabbit_url(
service_vars, protocol="pyamqp")
elif broker == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
self.celery_app.conf.broker_use_ssl = False
self.celery_app.conf.broker_url = self.get_redis_url(
service_vars, protocol="redis", db=RedisExt.CELERY_BROKER_DB)
else: # pragma: no cover
print_and_exit(
"Unable to start Celery: unknown broker service: {}", broker)
log.info(
"Configured {} as broker {}",
broker,
obfuscate_url(self.celery_app.conf.broker_url),
)
# From the guide: "Default: Taken from broker_url."
# But it is not true, connection fails if not explicitly set
self.celery_app.conf.broker_read_url = self.celery_app.conf.broker_url
self.celery_app.conf.broker_write_url = self.celery_app.conf.broker_url
backend = variables.get("backend_service", broker)
if backend == "RABBIT":
service_vars = Env.load_variables_group(prefix="rabbitmq")
log.warning(
"RABBIT backend is quite limited and not fully supported. "
"Consider to enable Redis as a backend database")
self.celery_app.conf.result_backend = self.get_rabbit_url(
service_vars, protocol="rpc")
elif backend == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
self.celery_app.conf.result_backend = self.get_redis_url(
service_vars, protocol="redis", db=RedisExt.CELERY_BACKEND_DB)
# set('redis_backend_use_ssl', kwargs.get('redis_backend_use_ssl'))
else: # pragma: no cover
print_and_exit(
"Unable to start Celery: unknown backend service: {}", backend)
log.info(
"Configured {} as backend {}",
backend,
obfuscate_url(self.celery_app.conf.result_backend),
)
# Should be enabled?
# Default: Disabled by default (transient messages).
# If set to True, result messages will be persistent.
# This means the messages won’t be lost after a broker restart.
# self.celery_app.conf.result_persistent = True
# Decides if publishing task messages will be retried in the case of
# connection loss or other connection errors
self.celery_app.conf.task_publish_retry = True
# Already enabled by default to use UTC
# self.celery_app.conf.enable_utc
# self.celery_app.conf.timezone
# Not needed, because tasks are dynamcally injected
# self.celery_app.conf.imports
# self.celery_app.conf.includes
# Note about priority: multi-queues is better than prioritized tasks
# https://docs.celeryproject.org/en/master/faq.html#does-celery-support-task-priorities
# Max priority default value for all queues
# Required to be able to set priority parameter on task calls
self.celery_app.conf.task_queue_max_priority = 10
# Default priority for taks (if not specified)
self.celery_app.conf.task_default_priority = 5
# If you want to apply a more strict priority to items
# probably prefetching should also be disabled:
# Late ack means the task messages will be acknowledged after the task
# has been executed, not just before (the default behavior).
# self.celery_app.conf.task_acks_late = True
# How many messages to prefetch at a time multiplied by the number
# of concurrent processes. The default is 4 (four messages for each process).
# The default setting is usually a good choice, however – if you have very
# long running tasks waiting in the queue and you have to start the workers,
# note that the first worker to start will receive four times the number
# of messages initially. Thus the tasks may not be fairly distributed to
# the workers. To disable prefetching, set worker_prefetch_multiplier to 1.
# Changing that setting to 0 will allow the worker to keep consuming as many
# messages as it wants.
self.celery_app.conf.worker_prefetch_multiplier = 1
# Introduced in Celery 5.1: on connection loss cancels all currently executed
# tasks with late acknowledgement enabled.
# These tasks cannot be acknowledged as the connection is gone,
# and the tasks are automatically redelivered back to the queue.
# In Celery 5.1 it is set to False by default.
# The setting will be set to True by default in Celery 6.0.
self.celery_app.conf.worker_cancel_long_running_tasks_on_connection_loss = True
if not PRODUCTION:
# Skip initial warnings by avoiding pickle format (deprecated)
# Only set in DEV mode since in PROD mode the auth serializer is used
self.celery_app.conf.accept_content = ["json"]
self.celery_app.conf.task_serializer = "json"
self.celery_app.conf.result_serializer = "json"
if Env.get_bool("CELERYBEAT_ENABLED"):
CeleryExt.CELERYBEAT_SCHEDULER = backend
if backend == "REDIS":
service_vars = Env.load_variables_group(prefix="redis")
url = self.get_redis_url(service_vars,
protocol="redis",
db=RedisExt.CELERY_BEAT_DB)
self.celery_app.conf["REDBEAT_REDIS_URL"] = url
self.celery_app.conf["REDBEAT_KEY_PREFIX"] = REDBEAT_KEY_PREFIX
log.info("Celery-beat connected to Redis: {}",
obfuscate_url(url))
else: # pragma: no cover
log.warning(
"Cannot configure celery beat scheduler with backend: {}",
backend)
# self.disconnected = False
conf = self.celery_app.conf
# Replace the previous App with new settings
self.celery_app = Celery("RAPyDo",
broker=conf.broker_url,
backend=conf.result_backend)
self.celery_app.conf = conf
if PRODUCTION:
# https://docs.celeryq.dev/en/stable/userguide/security.html#message-signing
self.celery_app.conf.update(
security_key=SSL_SECRET,
security_certificate=SSL_CERTIFICATE,
security_cert_store=SSL_CERTIFICATE,
security_digest="sha256",
task_serializer="auth",
result_serializer="auth",
event_serializer="auth",
accept_content=["auth"],
)
self.celery_app.setup_security()
for funct in Meta.get_celery_tasks(f"{CUSTOM_PACKAGE}.tasks"):
# Weird errors due to celery-stubs?
# "Callable[[], Any]" has no attribute "register"
# The code is correct... let's ignore it
self.celery_app.tasks.register(funct) # type: ignore
return self