def start_messaging():
app_channel.basic_consume(JOB_QUEUE, receive_job)
try:
log.info("RabbitMQ channel start consuming")
app_channel.start_consuming()
except KeyboardInterrupt:
app_channel.stop_consuming()
def timed(*args, **kw):
start = time.perf_counter()
result = method(*args, **kw)
end = time.perf_counter()
log.info(
f"{method.__name__} finished in {round(end - start, 4)} seconds")
return result
def __init__(self):
log.info(f"connecting redis using url: {REDIS_URL}")
self.co_occurrence = redis.ConnectionPool.from_url(url=REDIS_URL +
"/0")
self.job_search_pool = redis.ConnectionPool.from_url(url=REDIS_URL +
"/1")
self.job_keyword_pool = redis.ConnectionPool.from_url(url=REDIS_URL +
"/2")
self.standard_word_pool = redis.ConnectionPool.from_url(url=REDIS_URL +
"/3")
self.standard_category_pool = redis.ConnectionPool.from_url(
url=REDIS_URL + "/4")
def process_job_keywords(job_map: dict) -> Any:
"""Get the job keyword and publish it to mq"""
job_id = job_map["jobId"]
description = job_map["jobDescriptionText"]
request_id = job_map["requestId"]
job_number = job_map["jobNumber"]
total_job_count = job_map["totalJobCount"]
request_end = job_map["requestEnd"]
job_keyword_dto = generate_job_keyword(job_id, description)
# set request id to job keywords and publish to mq
job_keyword_dto.request_id = request_id
job_keyword_dto.job_number = job_number
job_keyword_dto.total_job_count = total_job_count
# when all jobs of this request are complete, send an ending message
if request_end is True or job_number == total_job_count:
job_keyword_dto.request_end = True
publish(job_keyword_dto.to_json())
log.info(f"published {job_number}/{total_job_count}")
import pika
from src.logger.logger import log
from src.setting.settings import RABBITMQ_URL
KEYWORD_EXCHANGE = "keyword-exchange"
KEYWORD_QUEUE = "keyword-queue"
KEYWORD_KEY = "keyword-key"
JOB_EXCHANGE = "job-exchange"
JOB_QUEUE = "job-queue"
JOB_KEY = "job-key"
log.info(f"connecting rabbitmq using url: {RABBITMQ_URL}")
conn_params = pika.URLParameters(RABBITMQ_URL)
connection = pika.BlockingConnection(conn_params)
app_channel = connection.channel()
app_channel.exchange_declare(KEYWORD_EXCHANGE,
durable=True,
exchange_type="direct")
app_channel.queue_declare(KEYWORD_QUEUE,
durable=True,
arguments={"x-message-ttl": 10000})
app_channel.queue_bind(KEYWORD_QUEUE, KEYWORD_EXCHANGE, KEYWORD_KEY)
app_channel.exchange_declare(JOB_EXCHANGE,
durable=True,
exchange_type="direct")
app_channel.queue_declare(JOB_QUEUE,
durable=True,
from sqlalchemy import create_engine
from src.logger.logger import log
from src.setting.settings import SQL_URL
log.info(f"connecting sql using url: {SQL_URL}")
conn = create_engine(SQL_URL, pool_recycle=3600)
def init_logging() -> None:
log.info(
f"** Your starting script path is {pathlib.Path(__file__).parent.absolute()}"
)
log.info(
f"** Your OS name is: {os.name}, {platform.system()}, {platform.release()}"
)
log.info(
f"** The version of Python you are running is: {platform.python_version()}"
)
log.info(f"** Your user home directory is: {Path.home()}")
log.info(f"** Your Python installation directory is: {sys.executable}")
log.info(f"** Amount of Your CPU cores: {os.cpu_count()}")
import pathlib
import platform
import sys
from pathlib import Path
import src.message.consumer
from src.logger.logger import log
ENV = "dev"
def init_logging() -> None:
log.info(
f"** Your starting script path is {pathlib.Path(__file__).parent.absolute()}"
)
log.info(
f"** Your OS name is: {os.name}, {platform.system()}, {platform.release()}"
)
log.info(
f"** The version of Python you are running is: {platform.python_version()}"
)
log.info(f"** Your user home directory is: {Path.home()}")
log.info(f"** Your Python installation directory is: {sys.executable}")
log.info(f"** Amount of Your CPU cores: {os.cpu_count()}")
if __name__ == '__main__':
log.info(f"app starting")
init_logging()
src.message.consumer.start_messaging()