def __init__(self, *args, **kwargs):
self.offset_reset = kwargs.pop("offset_reset", False)
super().__init__(*args, **kwargs)
self.logger = logging.getLogger("tamarco.kafka")
self.kafka_connection = None
self.kafka_client = None
self.task_manager = TasksManager(
task_limit=CONSUMER_PARALLEL_TASK_LIMIT)
self.loop = asyncio.get_event_loop()
def tasks_manager(event_loop):
tasks_manager = TasksManager()
tasks_manager.set_loop(event_loop)
yield tasks_manager
tasks_manager.stop_all()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = logging.getLogger("tamarco.amqp")
self.amqp_connection = None
self.task_manager = TasksManager()
self.loop = asyncio.get_event_loop()
class AMQPResource(IOResource):
depends_on = []
loggers_names = ["tamarco.amqp"]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = logging.getLogger("tamarco.amqp")
self.amqp_connection = None
self.task_manager = TasksManager()
self.loop = asyncio.get_event_loop()
async def get_rabbitmq_configuration(self):
rabbitmq_config = {
"host": await self.settings.get("host"),
"port": await self.settings.get("port"),
"vhost": await self.settings.get("vhost"),
"user": await self.settings.get("user"),
"password": await self.settings.get("password"),
"connection_timeout": await
self.settings.get("connection_timeout"),
"queues_prefix": await self.settings.get("queues_prefix", ""),
}
return rabbitmq_config
async def bind(self, *args, **kwargs):
await super().bind(*args, **kwargs)
self.task_manager.set_loop(self.microservice.loop)
self.loop = self.microservice.loop
async def connect_to_amqp(self):
"""Try to connect to the AMQP broker."""
self._status = StatusCodes.CONNECTING
rabbitmq_config = await self.get_rabbitmq_configuration()
await self.set_queues_prefix_iostreams()
self.amqp_connection = AMQPConnection(
**rabbitmq_config,
ioloop=self.loop,
on_error_callback=self.on_error_callback)
await self.amqp_connection.connect(
rabbitmq_config["connection_timeout"])
async def on_error_callback(self, exception):
if self._status != StatusCodes.STOPPING:
self.logger.critical(
f"Received error callback from AMQP connection, reporting failed status in AMQP "
f"resource. Exception: {exception}")
self._status = StatusCodes.FAILED
@property
def io_streams(self):
return tuple(chain(self.outputs.values(), self.inputs.values()))
async def set_queues_prefix_iostreams(self):
rabbitmq_config = await self.get_rabbitmq_configuration()
prefix = rabbitmq_config["queues_prefix"]
for io_elements in self.io_streams:
io_elements.set_queues_prefix(prefix)
self.logger.info(f"AMQP queues prefix: {prefix}")
async def start(self):
try:
await self.connect_to_amqp()
except ConnectionError:
self.logger.critical(
"AMQP resource cannot connect to RabbitMQ. Reporting failed status",
exc_info=True)
self._status = StatusCodes.FAILED
else:
try:
await self.amqp_connection.register(*self.io_streams)
except Exception:
self.logger.critical(
"Error registering handlers in AMQP resource. Reporting failed status",
exc_info=True)
self._status = StatusCodes.FAILED
else:
for input_element in self.inputs.values():
self.register_on_message_callback_trigger_task(
input_element)
await super().start()
async def post_start(self):
self.task_manager.start_all()
await super().post_start()
def register_on_message_callback_trigger_task(self, new_input):
"""Register a task that reads the messages from amqp. This task triggers others tasks.
Args:
new_input: AMQP input where to read messages.
"""
input_name = new_input.name
task_name = f"callback_trigger_{input_name}"
if new_input.on_message_callback:
self.task_manager.register_task(
task_name, new_input.callback_trigger(self.task_manager))
async def stop(self):
self.logger.info(f"Stopping AMQP resource")
if self._status == StatusCodes.STARTED:
self._status = StatusCodes.STOPPING
await self.amqp_connection.close()
self.task_manager.stop_all()
await super().stop()
async def status(self):
status = {"status": self._status}
if self._status == StatusCodes.STARTED:
connection_status = self.amqp_connection.is_connected()
amqp_configuration = self.amqp_connection.conn_parameters
inputs = [str(amqp_input) for amqp_input in self.inputs]
outputs = [str(amqp_output) for amqp_output in self.outputs]
status.update({
"connection_status": connection_status,
"amqp_config": amqp_configuration,
"inputs": inputs,
"outputs": outputs,
})
return status
class KafkaResource(IOResource):
depends_on = []
loggers_names = ["tamarco.kafka", "confluent-kafka"]
def __init__(self, *args, **kwargs):
self.offset_reset = kwargs.pop("offset_reset", False)
super().__init__(*args, **kwargs)
self.logger = logging.getLogger("tamarco.kafka")
self.kafka_connection = None
self.kafka_client = None
self.task_manager = TasksManager(
task_limit=CONSUMER_PARALLEL_TASK_LIMIT)
self.loop = asyncio.get_event_loop()
async def bind(self, *args, **kwargs):
await super().bind(*args, **kwargs)
self.loop = self.microservice.loop
self.task_manager.set_loop(self.loop)
async def get_confluent_kafka_settings(self):
bootstrap_servers = await self.settings.get("bootstrap_servers")
confluent_kafka_settings = {"bootstrap_servers": bootstrap_servers}
if self.offset_reset:
confluent_kafka_settings.update(
{"auto.offset.reset": self.offset_reset})
return confluent_kafka_settings
async def connect_to_kafka(self, confluent_kafka_settings):
try:
self._status = StatusCodes.CONNECTING
self.kafka_connection = KafkaConnection(
on_error_callback=self.on_disconnect_callback,
loop=self.loop,
**confluent_kafka_settings)
except Exception:
self.logger.exception("Invalid Kafka resource settings")
raise
else:
await self.kafka_connection.connect()
def on_disconnect_callback(self, exception):
# Workaround for avoid false positives disconnections is active, Disconnect don't trigger a fail status.
# When the broker is really down usually confluent-kafka trigger two callbacks, 'Disconnected' and other
# reporting 'ALL_BROKER_DOWN' so the second one should shut down the microservice in the real world.
if "Receive failed: Disconnected" not in str(exception):
self.logger.error(
f"Error in kafka resource. Reporting failed status. Exception: {exception}"
)
self._status = StatusCodes.FAILED
else:
self.logger.warning(
"Workaround to avoid false disconnections is active. The exception 'Receive failed: "
"Disconnected' do not trigger a failed status code in the kafka resource"
)
@property
def io_streams(self):
return tuple(chain(self.outputs.values(), self.inputs.values()))
async def start(self):
confluent_kafka_settings = await self.get_confluent_kafka_settings()
try:
await self.connect_to_kafka(confluent_kafka_settings)
except ConnectionError:
self.logger.critical(
"Kafka resource cannot connect to Kafka broker. Reporting failed status"
)
self._status = StatusCodes.FAILED
else:
try:
await self.kafka_connection.register(*self.io_streams)
except Exception:
self.logger.critical(
"Unexpected exception registering handlers in Kafka resource. Reporting failed "
"status",
exc_info=True,
)
self._status = StatusCodes.FAILED
else:
for input_element in self.inputs.values():
self.register_input_callback_trigger_task(input_element)
if self._status != StatusCodes.FAILED:
self._status = StatusCodes.STARTED
async def post_start(self):
self.task_manager.start_all()
await super().post_start()
def register_input_callback_trigger_task(self, new_input):
input_name = new_input.name
task_name = f"callback_trigger_{input_name}"
if new_input.on_message_callback:
self.task_manager.register_task(
task_name, new_input.callback_trigger(self.task_manager))
async def stop(self):
self.logger.info(f"Stopping Kafka resource: {self.name}")
if self._status != StatusCodes.NOT_STARTED or self._status != StatusCodes.STOPPED:
self._status = StatusCodes.STOPPING
try:
await self.kafka_connection.close()
except Exception:
self.logger.warning(
"Unexpected exception stopping the Kafka connection",
exc_info=True)
self.task_manager.stop_all()
await super().stop()
async def status(self):
status = {"status": self._status}
if self._status == StatusCodes.STARTED:
kafka_config = self.kafka_connection.confluent_conf
inputs = [str(kafka_input) for kafka_input in self.inputs]
outputs = [str(kafka_output) for kafka_output in self.outputs]
status.update({
"kafka_config": kafka_config,
"inputs": inputs,
"outputs": outputs
})
return status
class MicroserviceBase(metaclass=Singleton):
# Name of the microservice, is used by the resources
# to report a name of service.
name = None
# Instance id of the microservice, name is shared
# among instances but the instance id is unique.
instance_id = uuid.uuid4()
# Name of the deploy, is used by the resources
# to report a deploy name, is loaded by settings.
deploy_name = None
# Loggers to be added by the application code.
extra_loggers_names = []
# Main event loop.
loop = asyncio.get_event_loop()
# Manager for task.
tasks_manager = TasksManager()
# Settings manager.
settings = Settings()
# Logging manager.
logging = Logging()
@property
def loggers_names(self):
"""All loggers used by the framework.
Returns:
list: list of loggers names used by the microservice.
"""
loggers = {"tamarco", "tamarco.tasks", "tamarco.settings", "asyncio"}
for resource in self.resources.values():
loggers.update(resource.loggers_names)
loggers.update(self.extra_loggers_names)
loggers.update({self.name})
return loggers
def __new__(cls, *args, **kwargs):
cls.resources = OrderedDict()
dependency_graph = {
attr_name: getattr(cls, attr_name).depends_on
for attr_name in dir(cls)
if isinstance(getattr(cls, attr_name), BaseResource)
}
try:
resources_dep_ordered = resolve_dependency_order(dependency_graph)
except CantSolveDependencies as e:
print(e, file=sys.stderr)
exit(12)
else:
for name in resources_dep_ordered:
cls.resources[name] = getattr(cls, name)
return super().__new__(cls, *args, **kwargs)
def __init__(self):
assert self.name is not None, "Error, name should be defined in your microservice class"
self.logger = None
self._configure_provisional_logger()
def _configure_provisional_logger(self):
"""Provisional logging used before be able to read the final configuration from the settings."""
self.logger = logging.getLogger(self.name)
stdout_handler = logging.StreamHandler(sys.stdout)
print(
f"Configuring logger provisional logger of {self.name} to INFO and stdout"
)
self.logger.setLevel(logging.INFO)
self.logger.addHandler(stdout_handler)
self.logger.info(f"Configured {self.name} logger")
async def bind(self):
"""Call the bind function of all the resources.
It binds the resources to the microservice, allowing to the resources to identify their microservice.
"""
self.logger.info(
f"Binding to microservice the resources: {list(self.resources.keys())}"
)
await self.settings.bind(self.loop)
for name, resource in self.resources.items():
try:
await resource.bind(self, name)
except Exception:
self.logger.exception(
f"Unexpected exception binding the resource {resource}")
exit(11)
async def run_in_all_resources(self, method):
"""Run the method name in all the resources.
Args:
method (str): Method name to run in all the resources.
"""
for resource in self.resources.values():
self.logger.debug(f"Calling {method} of resource {resource.name}")
try:
await getattr(resource, method)()
except Exception:
self.logger.exception(
f"Error in {method} of resource {resource}")
else:
if method == "start":
self.logger.info(
f"Started {resource.name} from {self.name}")
async def start_logging(self):
"""Initializes the logging of the microservice."""
self.logger.info(
f"Starting logging in microservice {self.name} with loggers: {self.loggers_names}"
)
await self.logging.start(loggers=self.loggers_names,
microservice_name=self.name,
deploy_name=self.deploy_name,
loop=self.loop)
Informer.log_all_info(self.logger)
async def stop_settings(self):
"""Stops the settings of the microservice."""
self.logger.info("Stopping microservice settings")
await self.settings.stop()
async def start_settings(self):
"""Initializes the settings of the microservice."""
self.logger.info("Starting microservice settings")
await self.settings.start()
self.deploy_name = await self.settings.get(
f"{ROOT_SETTINGS}.deploy_name")
await self._configure_logging_settings()
await self._configure_resource_settings()
async def _configure_logging_settings(self):
self.logger.info("Configuring logging settings")
self.logging.configure_settings(
SettingsView(self.settings, f"{ROOT_SETTINGS}.logging", self.name))
async def _configure_resource_settings(self):
self.logger.info("Configuring resources settings")
for resource in self.resources.values():
await resource.configure_settings(
SettingsView(self.settings,
f"{ROOT_SETTINGS}.resources.{resource.name}",
self.name))
def _collect_tasks(self):
for attr_name in dir(self):
attr = getattr(self, attr_name)
if hasattr(attr, "_mark_task"):
self.tasks_manager.register_task(attr._name, attr)
elif hasattr(attr, "_mark_thread"):
self.tasks_manager.register_thread(attr._name, attr)