def process_response(self, req: Request, resp: Response,
resource: BaseController, _: bool) -> None:
# check if the controller allow logging
if resource is not None and resource.bypass_logger_middleware:
return
_res_size = 1
if resp is not None and resp.body is not None:
_res_size = len(resp.body)
log = {
'app':
'pubsub',
'date':
str(datetime.now()),
'reqId':
resp.get_header(REQUEST_ID),
'resCode':
int(resp.status[:3]),
'resTime':
math.ceil((time.time() - float(resp.get_header('timer'))) * 1000),
'resSize':
_res_size,
'method':
req.method,
'route':
req.relative_uri,
'details': {}
}
# if there is an error, add the message to the log
if self._is_info_response(resp):
try:
body = json.loads(resp.body)
body['payload'] = req.json
except (TypeError, AttributeError, JSONDecodeError):
body = {}
log['details'] = body
log['level'] = 'error'
LoggerUtils.error(json.dumps(log))
else:
# generate stdout
log['level'] = 'info'
log['details'] = req.json
LoggerUtils.info(json.dumps(log))
def create(self, name: str) -> None:
"""
Create an SQS queue
IMPORTANT: Make sure that your AWS have credential to create resource like an SQS bucket
:return: None
"""
try:
self.get_client().create_queue(QueueName=name)
LoggerUtils.info('Internal queue creation... \x1b[32mCOMPLETED')
assert self.get_client().list_queues()
except ClientError:
raise QueueException('Unable to create queue `{}`'.format(name))
def _build_log(service: BaseConsumer, body: Dict, response: Dict) -> Dict:
"""
Format log
:param service: BaseConsumer
:param body: dict
:param response: dict
:return: dict
"""
log = {
'app': 'pubsub-consumer',
'date': str(datetime.now()),
'reqId': json.loads(body.get('Message')).get('metadata', {}).get(REQUEST_ID),
'service': service.__class__.__name__,
'details': response
}
LoggerUtils.info(log)
def create(self, name: str) -> None:
"""
Create S3 bucket
IMPORTANT: Make sure that your AWS have credential to create resource like an S3 bucket
:return: None
"""
try:
self.get_client().create_bucket(
Bucket=name,
CreateBucketConfiguration={'LocationConstraint': AWS_REGION},
ObjectLockEnabledForBucket=False)
assert self.get_client().list_buckets() is not None
LoggerUtils.info('S3 bucket(s) created... \x1b[32mCOMPLETED')
except self.get_client().exceptions.BucketAlreadyExists:
pass
def process(service: BaseConsumer) -> None:
"""
Process messages from queue define in the service
:param service: BaseConsumer
:return: None
"""
while True:
try:
messages = service.pull_messages()
if messages is not None and 'Messages' in messages.keys():
LoggerUtils.debug('Total message pull :: {}'.format(len(messages.get('Messages'))))
for message in messages.get('Messages'):
body = json.loads(message.get('Body', {}))
response = service.distribute(body.get('Message'))
_build_log(service, body, response)
service.delete(message.get('ReceiptHandle'))
except (TypeError, BaseException) as e:
LoggerUtils.error(str(e))
import inspect
import os
import sys
# Add parent folder to the python path to be able to access "/application" and "/configs"
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from application.services.bucket import BucketService
from application.utils.logger import LoggerUtils
from configs.vars import AWS_S3_NAME
LoggerUtils.get_instance()
LoggerUtils.info('\x1b[33mCreate internal bucket')
# creating the bucket
BucketService().create(AWS_S3_NAME)
from __future__ import annotations
from typing import Any
import falcon
from falcon import Response
from application.controllers.healthcheck import HealthCheckController
from application.utils.logger import LoggerUtils
from application.middlewares.jsonify import JsonifyMiddleware
from application.controllers.messages import MessagesController
from application.middlewares.logger import LoggerMiddleware
from application.middlewares.request_id import RequestIdMiddleware
LoggerUtils.get_instance()
# Elasticsearch
LoggerUtils.info('\x1b[33mStarting API')
# Serializes all error messages as JSON before sending them.
def json_serializer(_, resp: Response, exception: Any):
resp.body = exception.to_json()
# Falcon
api = falcon.API(middleware=[
JsonifyMiddleware(),
LoggerMiddleware(),
RequestIdMiddleware()
import inspect
import os
import sys
# Add parent folder to the python path to be able to access "/application" and "/configs"
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from application.models.enums.protocol import SnsProtocol
from application.services.topic import TopicService
from application.utils.logger import LoggerUtils
from configs.vars import AWS_SNS_TOPIC, AWS_SQS_STREAMING, AWS_SQS_STORAGE
LoggerUtils.get_instance()
LoggerUtils.info('\x1b[33mCreate subscriptions')
# Queue to redirect to SNS for streaming
TopicService().subscriptions(SnsProtocol.SQS.value, AWS_SQS_STREAMING,
AWS_SNS_TOPIC)
LoggerUtils.info('\x1b[33mSubscriptions {} - {} and {} completed'.format(
SnsProtocol.SQS.value, AWS_SQS_STREAMING, AWS_SNS_TOPIC))
# Queue to send to the storage
TopicService().subscriptions(SnsProtocol.SQS.value, AWS_SQS_STORAGE,
AWS_SNS_TOPIC)
LoggerUtils.info('\x1b[33mSubscriptions {} - {} and {} completed'.format(
SnsProtocol.SQS.value, AWS_SQS_STORAGE, AWS_SNS_TOPIC))
# ======================================================================================
from application.models.enums.consumer import ConsumerEnum
from application.services.consumers.storage import StorageConsumerService
from application.services.consumers.base import BaseConsumer
from application.services.consumers.streaming import StreamingConsumerService
from application.exceptions.consumer import ConsumerException
from application.utils.logger import LoggerUtils
# Logger instance
from configs.vars import REQUEST_ID
LoggerUtils.get_instance()
def get_service(consumer: str) -> BaseConsumer:
"""
Based on the consumer, return instance of the service.
:param consumer: str
:return: StreamingConsumerService, StorageConsumerService
"""
if consumer == ConsumerEnum.STREAMING.value:
return StreamingConsumerService()
if consumer == ConsumerEnum.STORAGE.value:
return StorageConsumerService()
import inspect
import os
import sys
# Add parent folder to the python path to be able to access "/application" and "/configs"
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from application.services.topic import TopicService
from application.utils.logger import LoggerUtils
from configs.vars import AWS_SNS_TOPIC
LoggerUtils.get_instance()
LoggerUtils.info('\x1b[33mCreate internal topic')
TopicService().create(AWS_SNS_TOPIC)
# ======================================================================================
# This is just to create topic in a fast way.
# This should be done with Terraform
TopicService().create('Products.Brand')
TopicService().create('Products.Category')
TopicService().create('Products.Product')
import inspect
import os
import sys
# Add parent folder to the python path to be able to access "/application" and "/configs"
currentdir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from application.utils.logger import LoggerUtils
LoggerUtils.get_instance()
LoggerUtils.info('\x1b[33mCreate internal stream')
import inspect
import os
import sys
# Add parent folder to the python path to be able to access "/application" and "/configs"
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)
from configs.vars import AWS_SQS_STREAMING, AWS_SQS_STORAGE
from application.services.queue import QueueService
from application.utils.logger import LoggerUtils
LoggerUtils.get_instance()
LoggerUtils.info('\x1b[33mCreate internal queues')
QueueService().create(AWS_SQS_STREAMING)
QueueService().create(AWS_SQS_STORAGE)
# ======================================================================================
# This is just to create queue in a fast way.
# This should be done with Terraform
QueueService().create('service-consumer-queue')