def rabbitmq_broker(): broker = RabbitmqBroker( host="127.0.0.1", max_priority=10, credentials=RABBITMQ_CREDENTIALS, ) check_rabbitmq(broker) broker.emit_after("process_boot") dramatiq.set_broker(broker) yield broker broker.flush_all() broker.close()
def setup_worker(): """Set up Dramatiq with RabbitMQ. Dramatiq manages the message passing to background workers which run long tasks to avoid stalling the application responses for too long. """ logger.info('creating tasks queue') broker = RabbitmqBroker(url=f'{settings.RABBITMQ_URL}') dramatiq.set_broker(broker)
def test_rabbitmq_broker_can_be_passed_a_list_of_uri_for_failover(): # Given a string with a list of RabbitMQ connection URIs, including an invalid one # When I pass those URIs to RabbitMQ broker as a list broker = RabbitmqBroker(url=[ "amqp://127.0.0.1:55672", "amqp://%s:%[email protected]" % (RABBITMQ_USERNAME, RABBITMQ_PASSWORD) ]) # The the broker should connect to the host that is up assert broker.connection
def test_rabbitmq_broker_can_be_passed_a_list_of_parameters_for_failover(): # Given a list of pika connection parameters including an invalid one parameters = [ dict(host="127.0.0.1", port=55672), # this will fail dict(host="127.0.0.1"), ] # When I pass those parameters to RabbitmqBroker broker = RabbitmqBroker(parameters=parameters) # Then I should still get a connection to the host that is up assert broker.connection
def test_rabbitmq_broker_raises_an_error_if_given_invalid_parameter_combinations( ): # Given that I have a RabbitmqBroker # When I try to give it both a connection URL and a list of connection parameters # Then a RuntimeError should be raised with pytest.raises(RuntimeError): RabbitmqBroker(url="amqp://127.0.0.1:5672", parameters=[ dict(host="127.0.0.1", credentials=RABBITMQ_CREDENTIALS) ]) # When I try to give it both a connection URL and pika connection parameters # Then a RuntimeError should be raised with pytest.raises(RuntimeError): RabbitmqBroker(host="127.0.0.1", url="amqp://127.0.0.1:5672") # When I try to give it both a list of parameters and individual flags # Then a RuntimeError should be raised with pytest.raises(RuntimeError): RabbitmqBroker(host="127.0.0.1", parameters=[dict(host="127.0.0.1")])
def test_rabbitmq_broker_connections_are_lazy(): # When I create an RMQ broker broker = RabbitmqBroker( host="127.0.0.1", max_priority=10, credentials=RABBITMQ_CREDENTIALS, ) def get_connection(): return getattr(broker.state, "connection", None) # Then it shouldn't immediately connect to the server assert get_connection() is None # When I declare a queue broker.declare_queue("some-queue") # Then it shouldn't connect either assert get_connection() is None # When I create a consumer on that queue broker.consume("some-queue", timeout=1) # Then it should connect assert get_connection() is not None
def confirm_delivery_rabbitmq_broker(): broker = RabbitmqBroker( host="127.0.0.1", max_priority=10, credentials=RABBITMQ_CREDENTIALS, confirm_delivery=True, ) check_rabbitmq(broker) broker.emit_after("process_boot") dramatiq.set_broker(broker) yield broker try: broker.flush_all() except Exception: pass broker.close()
def get_dramatiq_broker_object(self): """ This method initializes the broker object for Dramatiq and saves it in Django's settings. """ if self.DRAMATIQ_BROKER_URL: url = self.DRAMATIQ_BROKER_URL.strip() if url is None: kind, host, port = "stub", None, None else: kind, _, url = url.partition("://") host, _, port = url.partition(":") host = host or None port = int(port) if port else None else: kind = self.DRAMATIQ_BROKER_TYPE host = self.DRAMATIQ_BROKER_HOST or None port = self.DRAMATIQ_BROKER_PORT or None # Separate non-null args kwargs = [("host", host), ("port", port)] kwargs = {k: v for k, v in kwargs if v is not None} # Initializes broker if kind == "stub": from dramatiq.brokers.stub import StubBroker broker = StubBroker() elif kind == "redis": from dramatiq.brokers.redis import RedisBroker broker = RedisBroker(**kwargs) elif kind == "rabbitmq": from dramatiq.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker(**kwargs) else: raise ValueError(f"invalid dramatiq broker: {kind}") # Configure as default and exit dramatiq.set_broker(broker) return broker
from dramatiq.brokers.redis import RedisBroker from dramatiq.brokers.rabbitmq import RabbitmqBroker logger = logging.getLogger("example") counter_key = "latench-bench-counter" memcache_client = pylibmc.Client(["localhost"], binary=True) memcache_pool = pylibmc.ClientPool(memcache_client, 8) random.seed(1337) if os.getenv("REDIS") == "1": broker = RedisBroker() dramatiq.set_broker(broker) celery_app = celery.Celery(broker="redis:///") else: broker = RabbitmqBroker() dramatiq.set_broker(broker) celery_app = celery.Celery(broker="amqp:///") def fib_bench(n): p, q = 0, 1 while n > 0: p, q = q, p + q n -= 1 with memcache_pool.reserve() as client: client.incr(counter_key) return p
from dataclasses import dataclass from pathlib import Path from typing import Any, Dict import dramatiq from dramatiq.brokers.rabbitmq import RabbitmqBroker from pony.orm import commit from pdf_server.database import Document, PdfStatus from pdf_server.exceptions import BadEntityRequestException from . import app __all__ = ["PdfManager"] rabbitmq_broker = RabbitmqBroker(**app.config["rabbitmq"]) dramatiq.set_broker(rabbitmq_broker) @dramatiq.actor def process_pdf(path: str) -> None: pass @dataclass class PdfInfo: status: PdfStatus n_pages: int def __post_init__(self): if not isinstance(self.status, PdfStatus):
import os import dramatiq import pika from dramatiq.brokers.rabbitmq import RabbitmqBroker # wrangle parameters credentials = pika.PlainCredentials(os.environ.get('RABBITMQ_USER'), os.environ.get('RABBITMQ_PASSWORD')) default_params = dict( virtual_host=os.environ.get('RABBITMQ_VHOST'), port=os.environ.get('RABBITMQ_PORT'), credentials=credentials, ) parameters = [ dict(host=host, **default_params) for host in os.environ.get('RABBITMQ_HOSTS').split(',') ] # initialize broker rabbitmq_broker = RabbitmqBroker(parameters=parameters) dramatiq.set_broker(rabbitmq_broker) # import the tasks from .shout import shout_something
import argparse import sys import requests import dramatiq from dramatiq import group from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.encoder import PickleEncoder from dramatiq.results import Results from dramatiq.results.backends import RedisBackend encoder = PickleEncoder() backend = RedisBackend(encoder=encoder) broker = RabbitmqBroker(host="127.0.0.1") broker.add_middleware(Results(backend=backend)) dramatiq.set_broker(broker) dramatiq.set_encoder(encoder) @dramatiq.actor def request(uri): return requests.get(uri) @dramatiq.actor(store_results=True) def count_words(response): return len(response.text.split(" ")) def main():
import os from typing import List from dacite import from_dict from judger_error import JudgerError from config import JUDGER_WORKSPACE, TEST_CASE_DIR from compile_config import CompileConfig from operation import Operation from problem_info import ProblemInfo from result import result from run_config import RunConfig from judger_task import Task result_backend = RedisBackend(url="redis://container.ll-ap.cn:6379") rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@container.ll-ap.cn:5672") rabbitmq_broker.add_middleware(Results(backend=result_backend)) # prometheus_middleware = Prometheus(http_host='0.0.0.0', http_port=9191) # rabbitmq_broker.add_middleware(prometheus_middleware) dramatiq.set_broker(rabbitmq_broker) @dramatiq.actor(queue_name='judge', store_results=True) def judge(task_id: str = None, problem_id: str = None, files: dict = None, operations: List[str] = None, config: dict = None): if task_id is None or problem_id is None: return
tika.initVM() # noqa # Esse bloco (feio) faz com que esse módulo funcione dentro ou fora do Django try: from web.datasets.models import File except ImproperlyConfigured: import configurations import os os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings") load_dotenv(find_dotenv()) configurations.setup() from web.datasets.models import File rabbitmq_broker = RabbitmqBroker(url=settings.BROKER_URL) set_broker(rabbitmq_broker) client = get_s3_client(settings) @actor(max_retries=5) def content_from_file(file_pk=None, path=None, keep_file=True): if not any([file_pk, path]): raise Exception("Ou `file_pk` ou `path` devem ser informados.") a_file = None if file_pk: a_file = File.objects.get(pk=file_pk) if a_file.content is not None: return a_file.content
import dramatiq import requests import json import uuid import os import shutil from compiler import Compiler from JudgeClient import JudgeClient from judger_config import TEST_CASE_DIR, JUDGER_WORKSPACE_BASE from dramatiq.brokers.rabbitmq import RabbitmqBroker from exception import CompileError import _judger from service_config import BROKER_URL, CALLBACK_URL, TOKEN rabbitmq_broker = RabbitmqBroker(url=BROKER_URL) dramatiq.set_broker(rabbitmq_broker) """ Request Data Structure { judge_id: String problem_id: String user_id: String code: String compiler: String max_cpu_time: Number max_memory: Number compiler_config: { src_name: String exe_name: String max_cpu_time: Number, max_real_time: Number,
import dramatiq import requests from dramatiq.brokers.rabbitmq import RabbitmqBroker def wc(url): response = requests.get(url) count = len(response.text.split(" ")) print(f"There are {count} words at {url!r}.") broker1 = RabbitmqBroker( host="rabbit", port=5672, heartbeat=5, connection_attempts=5, blocked_connection_timeout=30, ) # For it to work, just uncomment next line # dramatiq.broker.set_broker(broker1) class WordCounter(dramatiq.GenericActor): class Meta: broker = broker1 def perform(self, url): wc(url) @dramatiq.actor(broker=broker1)
import kizuna.plugins.kkreds import kizuna.plugins.ping from kizuna.adapters.slack import SlackAdapter from kizuna.skills import DB from kizuna.support import Kizuna from kizuna.support.dev_info import read_dev_info logging.basicConfig(level=logging.DEBUG) DEV_INFO = read_dev_info('./.dev-info.json') sentry = Client(config.SENTRY_URL, release=DEV_INFO.get('revision'), environment=config.KIZUNA_ENV) if config.SENTRY_URL else None rabbitmq_broker = RabbitmqBroker(url=config.RABBITMQ_URL) dramatiq.set_broker(rabbitmq_broker) if not config.SLACK_API_TOKEN: raise ValueError( 'You are missing a slack token! Please set the SLACK_API_TOKEN environment variable in your ' '.env file or in the system environment') sc = SlackClient(config.SLACK_API_TOKEN) db_engine = create_engine(config.DATABASE_URL) make_session = sessionmaker(bind=db_engine) k = Kizuna() k.adapters['slack'] = SlackAdapter(slack_client=sc)
def rabbitmq_broker(): broker = RabbitmqBroker() broker.emit_after("process_boot") dramatiq.set_broker(broker) yield broker broker.close()
import os import random import time import pytest import dramatiq from dramatiq.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker(host="127.0.0.1") @dramatiq.actor(queue_name="benchmark-throughput", broker=broker) def throughput(): pass @dramatiq.actor(queue_name="benchmark-fib", broker=broker) def fib(n): x, y = 1, 1 while n > 2: x, y = x + y, x n -= 1 return x @dramatiq.actor(queue_name="benchmark-latency", broker=broker) def latency(): p = random.randint(1, 100) if p == 1: durations = [3, 3, 3, 1]
# models precisam ser importados depois das configurações # para manter compatibilidade com o scraper from web.datasets.adapters import ( # noqa isort:skip to_citycouncil_bid, to_citycouncil_contract, to_citycouncil_expense, to_citycouncil_revenue, ) if os.getenv("DJANGO_CONFIGURATION") == "Test": broker = StubBroker() broker.emit_after("process_boot") else: broker = RabbitmqBroker( host=settings.BROKER_HOST, port=settings.BROKER_PORT, credentials=pika.credentials.PlainCredentials( settings.BROKER_USER, settings.BROKER_PASSWORD), virtual_host=settings.BROKER_VHOST, blocked_connection_timeout=300, ) set_broker(broker) client = get_s3_client(settings) class WebserviceException(Exception): pass @actor(max_retries=5) def content_from_file(file_pk=None, path=None, keep_file=True): if not any([file_pk, path]):
from models import db, Workload brokerService = os.getenv("DRAMATIQ_BROKER") config.load_incluster_config() apps_v1 = client.AppsV1Api() app = create_app() if brokerService.startswith('redis'): from dramatiq.brokers.redis import RedisBroker redis_broker = RedisBroker(url=brokerService) dramatiq.set_broker(redis_broker) else: from dramatiq.brokers.rabbitmq import RabbitmqBroker rabbitmq_broker = RabbitmqBroker(url=brokerService) dramatiq.set_broker(rabbitmq_broker) @dramatiq.actor def stop_workload(id, workload, workload_type, namespace): if workload_type == 'ReplicaSet': resp = apps_v1.list_namespaced_deployment(namespace=namespace) for i in resp.items: if i.metadata.name == workload: i.spec.replicas = 0 apps_v1.patch_namespaced_deployment(name=workload, namespace=namespace, body=i) elif workload_type == 'StatefulSet': resp = apps_v1.list_namespaced_stateful_set(namespace=namespace)
def init_dramatiq(): broker = RabbitmqBroker(url=config.rabbitmq.url) dramatiq.set_broker(broker)
def setup_broker(): print("Setting up broker...") broker = RabbitmqBroker() dramatiq.set_broker(broker)
from backend.settings import ( RABBITMQ_USER, RABBITMQ_HOST, RABBITMQ_PASSWORD, RABBITMQ_PORT, ) from social_auth.models import SocialProfile connection = {} if RABBITMQ_PASSWORD: connection[ "url" ] = f"amqp://{RABBITMQ_USER}:{RABBITMQ_PASSWORD}@{RABBITMQ_HOST}:{RABBITMQ_PORT}" rabbitmq_broker = RabbitmqBroker(**connection) dramatiq.set_broker(rabbitmq_broker) MINUTE = 60 * 1000 @dramatiq.actor(time_limit=10 * MINUTE) def test_long_running_request(): from core.models import User count = User.objects.filter(active=True).count() print(f"Active users: {count}") @dramatiq.actor()
from dramatiq.middleware import CurrentMessage from drama.config import settings from drama.database import get_db_connection from drama.logger import get_logger from drama.manager import TaskManager from drama.models.task import TaskResult, TaskStatus from drama.process import Process from drama.storage.helpers import get_available_storage from drama.worker.helpers import get_process_func logger = get_logger(__name__) # setup broker logger.debug("Setting up RabbitMQ broker") broker = RabbitmqBroker(url=settings.RABBIT_DNS) broker.add_middleware(CurrentMessage()) # set broker logger.debug("Attaching broker") dramatiq.set_broker(broker) @dramatiq.actor(**settings.DEFAULT_ACTOR_OPTS.dict()) def process_task(task_request: dict): """ Main `drama` actor. Executes an arbitrary function defined by a task and updates its state. """ message = CurrentMessage.get_current_message() task_id = message.message_id
from tika import parser # Esse bloco (feio) faz com que esse módulo funcione dentro ou fora do Django try: from datasets.models import File except ImproperlyConfigured: import configurations import os os.environ.setdefault("DJANGO_CONFIGURATION", "Dev") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") load_dotenv(find_dotenv()) configurations.setup() from datasets.models import File rabbitmq_broker = RabbitmqBroker(url=settings.BROKER_URL) rabbitmq_broker.add_middleware(middleware.Prometheus()) set_broker(rabbitmq_broker) client = get_s3_client(settings) @actor(max_retries=5) def content_from_file(file_pk=None, path=None, keep_file=True): if not any([file_pk, path]): raise Exception("Ou `file_pk` ou `path` devem ser informados.") a_file = None if file_pk: a_file = File.objects.get(pk=file_pk) if a_file.content is not None:
import argparse import dramatiq import requests import sys from dramatiq import group from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.encoder import PickleEncoder from dramatiq.results import Results from dramatiq.results.backends import RedisBackend encoder = PickleEncoder() backend = RedisBackend(encoder=encoder) broker = RabbitmqBroker() broker.add_middleware(Results(backend=backend)) dramatiq.set_broker(broker) dramatiq.set_encoder(encoder) @dramatiq.actor def request(uri): return requests.get(uri) @dramatiq.actor(store_results=True) def count_words(response): return len(response.text.split(" ")) def main(): parser = argparse.ArgumentParser()
import dramatiq import pika from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.results import Results from dramatiq.results.backends import RedisBackend result_backend_connection = "redis://*****:*****@redis_backend:6379/0" broker_connection = "amqp://*****:*****@rabbit:5672" result_backend = RedisBackend(url=result_backend_connection) rabbitmq_broker = RabbitmqBroker(url=broker_connection) rabbitmq_broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(rabbitmq_broker)
from typing import Optional, List from .config import BaseConfig config = BaseConfig() import dramatiq from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.middleware import CurrentMessage broker = RabbitmqBroker(host=config.RABBITMQ_HOST, middleware=[CurrentMessage()]) dramatiq.set_broker(broker) from fastapi import FastAPI from fastapi import Depends, FastAPI, HTTPException from fastapi.responses import HTMLResponse from fastapi.middleware.cors import CORSMiddleware from . import models, schemas from .database import session, engine, worker_session from .websockets import router models.BaseModel.metadata.create_all(bind=engine) app = FastAPI(docs_url='/') app.add_middleware( CORSMiddleware, allow_origins=["http://localhost:3000"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], )
# Firestore db access # Use a service account cred = credentials.Certificate('./eschernode-7d981168a58b.json') firebase_admin.initialize_app(cred) db = firestore.client() def get_user(uid): return auth.get_user(uid) if os.environ['ESCHERNODE_ENV'] == 'prod': es = elasticsearch.Elasticsearch(hosts=['172.30.0.151']) rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@172.30.0.11:5672") dashboard = 'https://eschernode.com/' else: es = elasticsearch.Elasticsearch() rabbitmq_broker = RabbitmqBroker() dashboard = 'http://127.0.0.1:3000/' # dramatiq.set_broker(rabbitmq_broker) # sparkpost # sp = sparkpost.SparkPost(SPARKPOST_API_KEY) # SES email client ses_email_client = boto3.client( 'ses', aws_access_key_id=AWS_ESCHERNODE_EMAIL_ACCESS_KEY,