def test_rabbitmq_broker_raises_an_error_if_given_invalid_parameter_combinations():
# Given that I have a RabbitmqBroker
# When I try to give it both a connection URL and a list of connection parameters
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(url="amqp://127.0.0.1:5672", parameters=[dict(host="127.0.0.1")])
# When I try to give it both a connection URL and pika connection parameters
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(host="127.0.0.1", url="amqp://127.0.0.1:5672")
# When I try to give it both a list of parameters and individual flags
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(host="127.0.0.1", parameters=[dict(host="127.0.0.1")])
def schedule(task_file, debug, rabbitmq, redis_url, expire):
try:
config = yaml.safe_load(task_file)
except yaml.YAMLError as e:
raise ClickException(f"Yaml task file is invalid: {e}")
if rabbitmq:
rabbitmq_broker = RabbitmqBroker(url=rabbitmq)
set_broker(rabbitmq_broker)
if debug:
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger("pika").setLevel(logging.CRITICAL)
scheduler = BlockingScheduler(timezone=utc)
try:
add_all_jobs(scheduler, config["jobs"])
except KeyError as e:
raise ClickException(
f"Config file missing required parameter: {e.args}")
if not redis_url:
scheduler.start()
return
conn = redis.Redis.from_url(redis_url)
with redis_lock.Lock(conn,
LOCK_NAME,
id=PROCESS_KEY,
expire=expire,
auto_renewal=True):
scheduler.start()
def test_rabbitmq_broker_can_be_passed_a_semicolon_separated_list_of_uris():
# Given a string with a list of RabbitMQ connection URIs, including an invalid one
# When I pass those URIs to RabbitMQ broker as a ;-separated string
broker = RabbitmqBroker(url="amqp://127.0.0.1:55672;amqp://127.0.0.1")
# The the broker should connect to the host that is up
assert broker.connection
def test_rabbitmq_broker_can_be_passed_a_list_of_uri_for_failover():
# Given a list of rabbitmq URI including an invalid one
# When I pass those URI on a string spplited by ';' to RabbitmqBroker
broker = RabbitmqBroker(url=['amqp://127.0.0.1:55672', 'amqp://127.0.0.1'])
# Then I should still get a connection to the host that is up
assert broker.connection
def test_rabbitmq_broker_can_be_passed_a_list_of_uri_for_failover():
# Given a string with a list of RabbitMQ connection URIs, including an invalid one
# When I pass those URIs to RabbitMQ broker as a list
broker = RabbitmqBroker(url=["amqp://127.0.0.1:55672", "amqp://127.0.0.1"])
# The the broker should connect to the host that is up
assert broker.connection
def get_broker() -> RabbitmqBroker:
"""Returns instance of rabbit MQ broker.
:returns: An instance of a Rabbit MQ broker.
"""
return RabbitmqBroker(url=_get_url())
def test_can_instantiate_brokers_without_middleware():
# Given that I have an empty list of middleware
# When I pass that to the RMQ Broker
broker = RabbitmqBroker(middleware=[])
# Then I should get back a broker with not middleware
assert not broker.middleware
def configure():
result_backend = RedisBackend(port=63791)
rabbitmq_broker = RabbitmqBroker(port=56721)
rabbitmq_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(rabbitmq_broker)
def test_rabbitmq_broker_connections_are_lazy():
# When I create an RMQ broker
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
)
def get_connection():
return getattr(broker.state, "connection", None)
# Then it shouldn't immediately connect to the server
assert get_connection() is None
# When I declare a queue
broker.declare_queue("some-queue")
# Then it shouldn't connect either
assert get_connection() is None
# When I create a consumer on that queue
broker.consume("some-queue", timeout=1)
# Then it should connect
assert get_connection() is not None
def rabbitmq_broker():
broker = RabbitmqBroker(host="127.0.0.1", max_priority=10)
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.flush_all()
broker.close()
def rabbitmq_broker():
broker = RabbitmqBroker()
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.flush_all()
broker.close()
def __init__(self):
# create RabbitMQ broker
self.broker = RabbitmqBroker(host=settings.MESSAGE_QUEUE_HOST,
port=settings.MESSAGE_QUEUE_PORT,
credentials=PlainCredentials(username=settings.MESSAGE_QUEUE_USERNAME,
password=settings.MESSAGE_QUEUE_PASSWORD))
# set the broker as default
set_broker(self.broker)
def setup_worker():
"""Set up Dramatiq with RabbitMQ.
Dramatiq manages the message passing to background workers which run
long tasks to avoid stalling the application responses for too long.
"""
logger.info('creating tasks queue')
broker = RabbitmqBroker(url=f'{settings.RABBITMQ_URL}')
dramatiq.set_broker(broker)
def __init__(self, sub_manager, config):
rabbitmq_host = config.get("RABBITMQ_HOST", None)
if rabbitmq_host is not None:
middleware = [
AgeLimit(), TimeLimit(), ShutdownNotifications(),
Callbacks(), Pipelines(), Retries()
]
self.broker = RabbitmqBroker(
host=rabbitmq_host, middleware=middleware)
else:
self.broker = get_broker()
def test_rabbitmq_broker_can_be_passed_a_list_of_parameters_for_failover():
# Given a list of pika connection parameters including an invalid one
parameters = [
dict(host="127.0.0.1", port=55672), # this will fail
dict(host="127.0.0.1"),
]
# When I pass those parameters to RabbitmqBroker
broker = RabbitmqBroker(parameters=parameters)
# Then I should still get a connection to the host that is up
assert broker.connection
def rabbitmq_broker():
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
)
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.flush_all()
broker.close()
def confirm_delivery_rabbitmq_broker():
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
confirm_delivery=True,
)
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
try:
broker.flush_all()
except Exception:
pass
broker.close()
def get_dramatiq_broker_object(self):
"""
This method initializes the broker object for Dramatiq and saves it in
Django's settings.
"""
if self.DRAMATIQ_BROKER_URL:
url = self.DRAMATIQ_BROKER_URL.strip()
if url is None:
kind, host, port = "stub", None, None
else:
kind, _, url = url.partition("://")
host, _, port = url.partition(":")
host = host or None
port = int(port) if port else None
else:
kind = self.DRAMATIQ_BROKER_TYPE
host = self.DRAMATIQ_BROKER_HOST or None
port = self.DRAMATIQ_BROKER_PORT or None
# Separate non-null args
kwargs = [("host", host), ("port", port)]
kwargs = {k: v for k, v in kwargs if v is not None}
# Initializes broker
if kind == "stub":
from dramatiq.brokers.stub import StubBroker
broker = StubBroker()
elif kind == "redis":
from dramatiq.brokers.redis import RedisBroker
broker = RedisBroker(**kwargs)
elif kind == "rabbitmq":
from dramatiq.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(**kwargs)
else:
raise ValueError(f"invalid dramatiq broker: {kind}")
# Configure as default and exit
dramatiq.set_broker(broker)
return broker
from typing import Optional, List
from .config import BaseConfig
config = BaseConfig()
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.middleware import CurrentMessage
broker = RabbitmqBroker(host=config.RABBITMQ_HOST,
middleware=[CurrentMessage()])
dramatiq.set_broker(broker)
from fastapi import FastAPI
from fastapi import Depends, FastAPI, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from . import models, schemas
from .database import session, engine, worker_session
from .websockets import router
models.BaseModel.metadata.create_all(bind=engine)
app = FastAPI(docs_url='/')
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:3000"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
import dramatiq
import requests
import json
import uuid
import os
import shutil
from compiler import Compiler
from JudgeClient import JudgeClient
from judger_config import TEST_CASE_DIR, JUDGER_WORKSPACE_BASE
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from exception import CompileError
import _judger
from service_config import BROKER_URL, CALLBACK_URL, TOKEN
rabbitmq_broker = RabbitmqBroker(url=BROKER_URL)
dramatiq.set_broker(rabbitmq_broker)
"""
Request Data Structure
{
judge_id: String
problem_id: String
user_id: String
code: String
compiler: String
max_cpu_time: Number
max_memory: Number
compiler_config: {
src_name: String
exe_name: String
max_cpu_time: Number,
max_real_time: Number,
import argparse
import dramatiq
import requests
import sys
from dramatiq import group
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
@dramatiq.actor
def request(uri):
return requests.get(uri)
@dramatiq.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
parser = argparse.ArgumentParser()
import os
from typing import List
from dacite import from_dict
from judger_error import JudgerError
from config import JUDGER_WORKSPACE, TEST_CASE_DIR
from compile_config import CompileConfig
from operation import Operation
from problem_info import ProblemInfo
from result import result
from run_config import RunConfig
from judger_task import Task
result_backend = RedisBackend(url="redis://container.ll-ap.cn:6379")
rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@container.ll-ap.cn:5672")
rabbitmq_broker.add_middleware(Results(backend=result_backend))
# prometheus_middleware = Prometheus(http_host='0.0.0.0', http_port=9191)
# rabbitmq_broker.add_middleware(prometheus_middleware)
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor(queue_name='judge', store_results=True)
def judge(task_id: str = None,
problem_id: str = None,
files: dict = None,
operations: List[str] = None,
config: dict = None):
if task_id is None or problem_id is None:
return
tika.initVM() # noqa
# Esse bloco (feio) faz com que esse módulo funcione dentro ou fora do Django
try:
from web.datasets.models import File
except ImproperlyConfigured:
import configurations
import os
os.environ.setdefault("DJANGO_CONFIGURATION", "Dev")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings")
load_dotenv(find_dotenv())
configurations.setup()
from web.datasets.models import File
rabbitmq_broker = RabbitmqBroker(url=settings.BROKER_URL)
set_broker(rabbitmq_broker)
client = get_s3_client(settings)
@actor(max_retries=5)
def content_from_file(file_pk=None, path=None, keep_file=True):
if not any([file_pk, path]):
raise Exception("Ou `file_pk` ou `path` devem ser informados.")
a_file = None
if file_pk:
a_file = File.objects.get(pk=file_pk)
if a_file.content is not None:
return a_file.content
def setup_broker():
print("Setting up broker...")
broker = RabbitmqBroker()
dramatiq.set_broker(broker)
def init_dramatiq():
broker = RabbitmqBroker(url=config.rabbitmq.url)
dramatiq.set_broker(broker)
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from pony.orm import commit
from pdf_server.database import Document, PdfStatus
from pdf_server.exceptions import BadEntityRequestException
from . import app
__all__ = ["PdfManager"]
rabbitmq_broker = RabbitmqBroker(**app.config["rabbitmq"])
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor
def process_pdf(path: str) -> None:
pass
@dataclass
class PdfInfo:
status: PdfStatus
n_pages: int
def __post_init__(self):
if not isinstance(self.status, PdfStatus):
# models precisam ser importados depois das configurações
# para manter compatibilidade com o scraper
from web.datasets.adapters import ( # noqa isort:skip
to_citycouncil_bid, to_citycouncil_contract, to_citycouncil_expense,
to_citycouncil_revenue,
)
if os.getenv("DJANGO_CONFIGURATION") == "Test":
broker = StubBroker()
broker.emit_after("process_boot")
else:
broker = RabbitmqBroker(
host=settings.BROKER_HOST,
port=settings.BROKER_PORT,
credentials=pika.credentials.PlainCredentials(
settings.BROKER_USER, settings.BROKER_PASSWORD),
virtual_host=settings.BROKER_VHOST,
blocked_connection_timeout=300,
)
set_broker(broker)
client = get_s3_client(settings)
class WebserviceException(Exception):
pass
@actor(max_retries=5)
def content_from_file(file_pk=None, path=None, keep_file=True):
if not any([file_pk, path]):
import os
import random
import time
import pytest
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(host="127.0.0.1")
@dramatiq.actor(queue_name="benchmark-throughput", broker=broker)
def throughput():
pass
@dramatiq.actor(queue_name="benchmark-fib", broker=broker)
def fib(n):
x, y = 1, 1
while n > 2:
x, y = x + y, x
n -= 1
return x
@dramatiq.actor(queue_name="benchmark-latency", broker=broker)
def latency():
p = random.randint(1, 100)
if p == 1:
durations = [3, 3, 3, 1]
import kizuna.plugins.kkreds
import kizuna.plugins.ping
from kizuna.adapters.slack import SlackAdapter
from kizuna.skills import DB
from kizuna.support import Kizuna
from kizuna.support.dev_info import read_dev_info
logging.basicConfig(level=logging.DEBUG)
DEV_INFO = read_dev_info('./.dev-info.json')
sentry = Client(config.SENTRY_URL,
release=DEV_INFO.get('revision'),
environment=config.KIZUNA_ENV) if config.SENTRY_URL else None
rabbitmq_broker = RabbitmqBroker(url=config.RABBITMQ_URL)
dramatiq.set_broker(rabbitmq_broker)
if not config.SLACK_API_TOKEN:
raise ValueError(
'You are missing a slack token! Please set the SLACK_API_TOKEN environment variable in your '
'.env file or in the system environment')
sc = SlackClient(config.SLACK_API_TOKEN)
db_engine = create_engine(config.DATABASE_URL)
make_session = sessionmaker(bind=db_engine)
k = Kizuna()
k.adapters['slack'] = SlackAdapter(slack_client=sc)
import dramatiq
import requests
from dramatiq.brokers.rabbitmq import RabbitmqBroker
def wc(url):
response = requests.get(url)
count = len(response.text.split(" "))
print(f"There are {count} words at {url!r}.")
broker1 = RabbitmqBroker(
host="rabbit",
port=5672,
heartbeat=5,
connection_attempts=5,
blocked_connection_timeout=30,
)
# For it to work, just uncomment next line
# dramatiq.broker.set_broker(broker1)
class WordCounter(dramatiq.GenericActor):
class Meta:
broker = broker1
def perform(self, url):
wc(url)
@dramatiq.actor(broker=broker1)