def rabbitmq_broker():
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
)
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.flush_all()
broker.close()
def setup_worker():
"""Set up Dramatiq with RabbitMQ.
Dramatiq manages the message passing to background workers which run
long tasks to avoid stalling the application responses for too long.
"""
logger.info('creating tasks queue')
broker = RabbitmqBroker(url=f'{settings.RABBITMQ_URL}')
dramatiq.set_broker(broker)
def test_rabbitmq_broker_can_be_passed_a_list_of_uri_for_failover():
# Given a string with a list of RabbitMQ connection URIs, including an invalid one
# When I pass those URIs to RabbitMQ broker as a list
broker = RabbitmqBroker(url=[
"amqp://127.0.0.1:55672",
"amqp://%s:%[email protected]" % (RABBITMQ_USERNAME, RABBITMQ_PASSWORD)
])
# The the broker should connect to the host that is up
assert broker.connection
def test_rabbitmq_broker_can_be_passed_a_list_of_parameters_for_failover():
# Given a list of pika connection parameters including an invalid one
parameters = [
dict(host="127.0.0.1", port=55672), # this will fail
dict(host="127.0.0.1"),
]
# When I pass those parameters to RabbitmqBroker
broker = RabbitmqBroker(parameters=parameters)
# Then I should still get a connection to the host that is up
assert broker.connection
def test_rabbitmq_broker_raises_an_error_if_given_invalid_parameter_combinations(
):
# Given that I have a RabbitmqBroker
# When I try to give it both a connection URL and a list of connection parameters
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(url="amqp://127.0.0.1:5672",
parameters=[
dict(host="127.0.0.1",
credentials=RABBITMQ_CREDENTIALS)
])
# When I try to give it both a connection URL and pika connection parameters
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(host="127.0.0.1", url="amqp://127.0.0.1:5672")
# When I try to give it both a list of parameters and individual flags
# Then a RuntimeError should be raised
with pytest.raises(RuntimeError):
RabbitmqBroker(host="127.0.0.1", parameters=[dict(host="127.0.0.1")])
def test_rabbitmq_broker_connections_are_lazy():
# When I create an RMQ broker
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
)
def get_connection():
return getattr(broker.state, "connection", None)
# Then it shouldn't immediately connect to the server
assert get_connection() is None
# When I declare a queue
broker.declare_queue("some-queue")
# Then it shouldn't connect either
assert get_connection() is None
# When I create a consumer on that queue
broker.consume("some-queue", timeout=1)
# Then it should connect
assert get_connection() is not None
def confirm_delivery_rabbitmq_broker():
broker = RabbitmqBroker(
host="127.0.0.1",
max_priority=10,
credentials=RABBITMQ_CREDENTIALS,
confirm_delivery=True,
)
check_rabbitmq(broker)
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
try:
broker.flush_all()
except Exception:
pass
broker.close()
def get_dramatiq_broker_object(self):
"""
This method initializes the broker object for Dramatiq and saves it in
Django's settings.
"""
if self.DRAMATIQ_BROKER_URL:
url = self.DRAMATIQ_BROKER_URL.strip()
if url is None:
kind, host, port = "stub", None, None
else:
kind, _, url = url.partition("://")
host, _, port = url.partition(":")
host = host or None
port = int(port) if port else None
else:
kind = self.DRAMATIQ_BROKER_TYPE
host = self.DRAMATIQ_BROKER_HOST or None
port = self.DRAMATIQ_BROKER_PORT or None
# Separate non-null args
kwargs = [("host", host), ("port", port)]
kwargs = {k: v for k, v in kwargs if v is not None}
# Initializes broker
if kind == "stub":
from dramatiq.brokers.stub import StubBroker
broker = StubBroker()
elif kind == "redis":
from dramatiq.brokers.redis import RedisBroker
broker = RedisBroker(**kwargs)
elif kind == "rabbitmq":
from dramatiq.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(**kwargs)
else:
raise ValueError(f"invalid dramatiq broker: {kind}")
# Configure as default and exit
dramatiq.set_broker(broker)
return broker
from dramatiq.brokers.redis import RedisBroker
from dramatiq.brokers.rabbitmq import RabbitmqBroker
logger = logging.getLogger("example")
counter_key = "latench-bench-counter"
memcache_client = pylibmc.Client(["localhost"], binary=True)
memcache_pool = pylibmc.ClientPool(memcache_client, 8)
random.seed(1337)
if os.getenv("REDIS") == "1":
broker = RedisBroker()
dramatiq.set_broker(broker)
celery_app = celery.Celery(broker="redis:///")
else:
broker = RabbitmqBroker()
dramatiq.set_broker(broker)
celery_app = celery.Celery(broker="amqp:///")
def fib_bench(n):
p, q = 0, 1
while n > 0:
p, q = q, p + q
n -= 1
with memcache_pool.reserve() as client:
client.incr(counter_key)
return p
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from pony.orm import commit
from pdf_server.database import Document, PdfStatus
from pdf_server.exceptions import BadEntityRequestException
from . import app
__all__ = ["PdfManager"]
rabbitmq_broker = RabbitmqBroker(**app.config["rabbitmq"])
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor
def process_pdf(path: str) -> None:
pass
@dataclass
class PdfInfo:
status: PdfStatus
n_pages: int
def __post_init__(self):
if not isinstance(self.status, PdfStatus):
import os
import dramatiq
import pika
from dramatiq.brokers.rabbitmq import RabbitmqBroker
# wrangle parameters
credentials = pika.PlainCredentials(os.environ.get('RABBITMQ_USER'),
os.environ.get('RABBITMQ_PASSWORD'))
default_params = dict(
virtual_host=os.environ.get('RABBITMQ_VHOST'),
port=os.environ.get('RABBITMQ_PORT'),
credentials=credentials,
)
parameters = [
dict(host=host, **default_params)
for host in os.environ.get('RABBITMQ_HOSTS').split(',')
]
# initialize broker
rabbitmq_broker = RabbitmqBroker(parameters=parameters)
dramatiq.set_broker(rabbitmq_broker)
# import the tasks
from .shout import shout_something
import argparse
import sys
import requests
import dramatiq
from dramatiq import group
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker(host="127.0.0.1")
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
@dramatiq.actor
def request(uri):
return requests.get(uri)
@dramatiq.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
import os
from typing import List
from dacite import from_dict
from judger_error import JudgerError
from config import JUDGER_WORKSPACE, TEST_CASE_DIR
from compile_config import CompileConfig
from operation import Operation
from problem_info import ProblemInfo
from result import result
from run_config import RunConfig
from judger_task import Task
result_backend = RedisBackend(url="redis://container.ll-ap.cn:6379")
rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@container.ll-ap.cn:5672")
rabbitmq_broker.add_middleware(Results(backend=result_backend))
# prometheus_middleware = Prometheus(http_host='0.0.0.0', http_port=9191)
# rabbitmq_broker.add_middleware(prometheus_middleware)
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor(queue_name='judge', store_results=True)
def judge(task_id: str = None,
problem_id: str = None,
files: dict = None,
operations: List[str] = None,
config: dict = None):
if task_id is None or problem_id is None:
return
tika.initVM() # noqa
# Esse bloco (feio) faz com que esse módulo funcione dentro ou fora do Django
try:
from web.datasets.models import File
except ImproperlyConfigured:
import configurations
import os
os.environ.setdefault("DJANGO_CONFIGURATION", "Dev")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings")
load_dotenv(find_dotenv())
configurations.setup()
from web.datasets.models import File
rabbitmq_broker = RabbitmqBroker(url=settings.BROKER_URL)
set_broker(rabbitmq_broker)
client = get_s3_client(settings)
@actor(max_retries=5)
def content_from_file(file_pk=None, path=None, keep_file=True):
if not any([file_pk, path]):
raise Exception("Ou `file_pk` ou `path` devem ser informados.")
a_file = None
if file_pk:
a_file = File.objects.get(pk=file_pk)
if a_file.content is not None:
return a_file.content
import dramatiq
import requests
import json
import uuid
import os
import shutil
from compiler import Compiler
from JudgeClient import JudgeClient
from judger_config import TEST_CASE_DIR, JUDGER_WORKSPACE_BASE
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from exception import CompileError
import _judger
from service_config import BROKER_URL, CALLBACK_URL, TOKEN
rabbitmq_broker = RabbitmqBroker(url=BROKER_URL)
dramatiq.set_broker(rabbitmq_broker)
"""
Request Data Structure
{
judge_id: String
problem_id: String
user_id: String
code: String
compiler: String
max_cpu_time: Number
max_memory: Number
compiler_config: {
src_name: String
exe_name: String
max_cpu_time: Number,
max_real_time: Number,
import dramatiq
import requests
from dramatiq.brokers.rabbitmq import RabbitmqBroker
def wc(url):
response = requests.get(url)
count = len(response.text.split(" "))
print(f"There are {count} words at {url!r}.")
broker1 = RabbitmqBroker(
host="rabbit",
port=5672,
heartbeat=5,
connection_attempts=5,
blocked_connection_timeout=30,
)
# For it to work, just uncomment next line
# dramatiq.broker.set_broker(broker1)
class WordCounter(dramatiq.GenericActor):
class Meta:
broker = broker1
def perform(self, url):
wc(url)
@dramatiq.actor(broker=broker1)
import kizuna.plugins.kkreds
import kizuna.plugins.ping
from kizuna.adapters.slack import SlackAdapter
from kizuna.skills import DB
from kizuna.support import Kizuna
from kizuna.support.dev_info import read_dev_info
logging.basicConfig(level=logging.DEBUG)
DEV_INFO = read_dev_info('./.dev-info.json')
sentry = Client(config.SENTRY_URL,
release=DEV_INFO.get('revision'),
environment=config.KIZUNA_ENV) if config.SENTRY_URL else None
rabbitmq_broker = RabbitmqBroker(url=config.RABBITMQ_URL)
dramatiq.set_broker(rabbitmq_broker)
if not config.SLACK_API_TOKEN:
raise ValueError(
'You are missing a slack token! Please set the SLACK_API_TOKEN environment variable in your '
'.env file or in the system environment')
sc = SlackClient(config.SLACK_API_TOKEN)
db_engine = create_engine(config.DATABASE_URL)
make_session = sessionmaker(bind=db_engine)
k = Kizuna()
k.adapters['slack'] = SlackAdapter(slack_client=sc)
def rabbitmq_broker():
broker = RabbitmqBroker()
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.close()
import os
import random
import time
import pytest
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(host="127.0.0.1")
@dramatiq.actor(queue_name="benchmark-throughput", broker=broker)
def throughput():
pass
@dramatiq.actor(queue_name="benchmark-fib", broker=broker)
def fib(n):
x, y = 1, 1
while n > 2:
x, y = x + y, x
n -= 1
return x
@dramatiq.actor(queue_name="benchmark-latency", broker=broker)
def latency():
p = random.randint(1, 100)
if p == 1:
durations = [3, 3, 3, 1]
# models precisam ser importados depois das configurações
# para manter compatibilidade com o scraper
from web.datasets.adapters import ( # noqa isort:skip
to_citycouncil_bid, to_citycouncil_contract, to_citycouncil_expense,
to_citycouncil_revenue,
)
if os.getenv("DJANGO_CONFIGURATION") == "Test":
broker = StubBroker()
broker.emit_after("process_boot")
else:
broker = RabbitmqBroker(
host=settings.BROKER_HOST,
port=settings.BROKER_PORT,
credentials=pika.credentials.PlainCredentials(
settings.BROKER_USER, settings.BROKER_PASSWORD),
virtual_host=settings.BROKER_VHOST,
blocked_connection_timeout=300,
)
set_broker(broker)
client = get_s3_client(settings)
class WebserviceException(Exception):
pass
@actor(max_retries=5)
def content_from_file(file_pk=None, path=None, keep_file=True):
if not any([file_pk, path]):
from models import db, Workload
brokerService = os.getenv("DRAMATIQ_BROKER")
config.load_incluster_config()
apps_v1 = client.AppsV1Api()
app = create_app()
if brokerService.startswith('redis'):
from dramatiq.brokers.redis import RedisBroker
redis_broker = RedisBroker(url=brokerService)
dramatiq.set_broker(redis_broker)
else:
from dramatiq.brokers.rabbitmq import RabbitmqBroker
rabbitmq_broker = RabbitmqBroker(url=brokerService)
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor
def stop_workload(id, workload, workload_type, namespace):
if workload_type == 'ReplicaSet':
resp = apps_v1.list_namespaced_deployment(namespace=namespace)
for i in resp.items:
if i.metadata.name == workload:
i.spec.replicas = 0
apps_v1.patch_namespaced_deployment(name=workload,
namespace=namespace,
body=i)
elif workload_type == 'StatefulSet':
resp = apps_v1.list_namespaced_stateful_set(namespace=namespace)
def init_dramatiq():
broker = RabbitmqBroker(url=config.rabbitmq.url)
dramatiq.set_broker(broker)
def setup_broker():
print("Setting up broker...")
broker = RabbitmqBroker()
dramatiq.set_broker(broker)
from backend.settings import (
RABBITMQ_USER,
RABBITMQ_HOST,
RABBITMQ_PASSWORD,
RABBITMQ_PORT,
)
from social_auth.models import SocialProfile
connection = {}
if RABBITMQ_PASSWORD:
connection[
"url"
] = f"amqp://{RABBITMQ_USER}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}"
rabbitmq_broker = RabbitmqBroker(**connection)
dramatiq.set_broker(rabbitmq_broker)
MINUTE = 60 * 1000
@dramatiq.actor(time_limit=10 * MINUTE)
def test_long_running_request():
from core.models import User
count = User.objects.filter(active=True).count()
print(f"Active users: {count}")
@dramatiq.actor()
from dramatiq.middleware import CurrentMessage
from drama.config import settings
from drama.database import get_db_connection
from drama.logger import get_logger
from drama.manager import TaskManager
from drama.models.task import TaskResult, TaskStatus
from drama.process import Process
from drama.storage.helpers import get_available_storage
from drama.worker.helpers import get_process_func
logger = get_logger(__name__)
# setup broker
logger.debug("Setting up RabbitMQ broker")
broker = RabbitmqBroker(url=settings.RABBIT_DNS)
broker.add_middleware(CurrentMessage())
# set broker
logger.debug("Attaching broker")
dramatiq.set_broker(broker)
@dramatiq.actor(**settings.DEFAULT_ACTOR_OPTS.dict())
def process_task(task_request: dict):
"""
Main `drama` actor.
Executes an arbitrary function defined by a task and updates its state.
"""
message = CurrentMessage.get_current_message()
task_id = message.message_id
from tika import parser
# Esse bloco (feio) faz com que esse módulo funcione dentro ou fora do Django
try:
from datasets.models import File
except ImproperlyConfigured:
import configurations
import os
os.environ.setdefault("DJANGO_CONFIGURATION", "Dev")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings")
load_dotenv(find_dotenv())
configurations.setup()
from datasets.models import File
rabbitmq_broker = RabbitmqBroker(url=settings.BROKER_URL)
rabbitmq_broker.add_middleware(middleware.Prometheus())
set_broker(rabbitmq_broker)
client = get_s3_client(settings)
@actor(max_retries=5)
def content_from_file(file_pk=None, path=None, keep_file=True):
if not any([file_pk, path]):
raise Exception("Ou `file_pk` ou `path` devem ser informados.")
a_file = None
if file_pk:
a_file = File.objects.get(pk=file_pk)
if a_file.content is not None:
import argparse
import dramatiq
import requests
import sys
from dramatiq import group
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
@dramatiq.actor
def request(uri):
return requests.get(uri)
@dramatiq.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
parser = argparse.ArgumentParser()
import dramatiq import pika from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.results import Results from dramatiq.results.backends import RedisBackend result_backend_connection = "redis://*****:*****@redis_backend:6379/0" broker_connection = "amqp://*****:*****@rabbit:5672" result_backend = RedisBackend(url=result_backend_connection) rabbitmq_broker = RabbitmqBroker(url=broker_connection) rabbitmq_broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(rabbitmq_broker)
from typing import Optional, List
from .config import BaseConfig
config = BaseConfig()
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.middleware import CurrentMessage
broker = RabbitmqBroker(host=config.RABBITMQ_HOST,
middleware=[CurrentMessage()])
dramatiq.set_broker(broker)
from fastapi import FastAPI
from fastapi import Depends, FastAPI, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from . import models, schemas
from .database import session, engine, worker_session
from .websockets import router
models.BaseModel.metadata.create_all(bind=engine)
app = FastAPI(docs_url='/')
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:3000"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Firestore db access
# Use a service account
cred = credentials.Certificate('./eschernode-7d981168a58b.json')
firebase_admin.initialize_app(cred)
db = firestore.client()
def get_user(uid):
return auth.get_user(uid)
if os.environ['ESCHERNODE_ENV'] == 'prod':
es = elasticsearch.Elasticsearch(hosts=['172.30.0.151'])
rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@172.30.0.11:5672")
dashboard = 'https://eschernode.com/'
else:
es = elasticsearch.Elasticsearch()
rabbitmq_broker = RabbitmqBroker()
dashboard = 'http://127.0.0.1:3000/'
# dramatiq.set_broker(rabbitmq_broker)
# sparkpost
# sp = sparkpost.SparkPost(SPARKPOST_API_KEY)
# SES email client
ses_email_client = boto3.client(
'ses',
aws_access_key_id=AWS_ESCHERNODE_EMAIL_ACCESS_KEY,