def ready(self):
dramatiq.set_encoder(self.select_encoder())
result_backend_settings = self.result_backend_settings()
if result_backend_settings:
result_backend_path = result_backend_settings.get(
"BACKEND", "dramatiq.results.backends.StubBackend")
result_backend_class = load_class(result_backend_path)
result_backend_options = result_backend_settings.get(
"BACKEND_OPTIONS", {})
result_backend = result_backend_class(**result_backend_options)
results_middleware_options = result_backend_settings.get(
"MIDDLEWARE_OPTIONS", {})
results_middleware = Results(backend=result_backend,
**results_middleware_options)
else:
result_backend = None
results_middleware = None
broker_settings = self.broker_settings()
broker_path = broker_settings["BROKER"]
broker_class = load_class(broker_path)
broker_options = broker_settings.get("OPTIONS", {})
middleware = [
load_middleware(path)
for path in broker_settings.get("MIDDLEWARE", [])
]
if result_backend is not None:
middleware.append(results_middleware)
broker = broker_class(middleware=middleware, **broker_options)
dramatiq.set_broker(broker)
def initialize_broker(broker, results_backend=None):
dramatiq.set_broker(broker)
dramatiq.set_encoder(middleware.TypedEncoder())
if results_backend:
results_middleware = dramatiq.results.Results(backend=results_backend)
broker.add_middleware(results_middleware)
progress_middleware = middleware.ProgressMiddleware(cache.client)
broker.add_middleware(progress_middleware)
def ready(self):
broker_settings = self.broker_settings()
broker_path = broker_settings["BROKER"]
broker_class = load_class(broker_path)
broker_options = broker_settings.get("OPTIONS", {})
middleware = [
load_middleware(path)
for path in broker_settings.get("MIDDLEWARE", [])
]
broker = broker_class(middleware=middleware, **broker_options)
dramatiq.set_broker(broker)
dramatiq.set_encoder(self.select_encoder())
def setup_redis_broker() -> None:
_connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool(
host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD
)
_redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool)
_result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db)
_result_middleware = Results(backend=_result_backend)
broker: Broker = RedisBroker(
connection_pool=_connection_pool,
middleware=[_result_middleware],
namespace="lrw",
)
dramatiq.set_broker(broker)
dramatiq.set_encoder(dramatiq.PickleEncoder())
def ready(self):
# this line should be first, to set up global encoder before any middleware is created
dramatiq.set_encoder(self.select_encoder())
broker_settings = self.broker_settings()
broker_path = broker_settings["BROKER"]
broker_class = load_class(broker_path)
broker_options = broker_settings.get("OPTIONS", {})
middleware = [
load_middleware(path)
for path in broker_settings.get("MIDDLEWARE", [])
]
broker = broker_class(middleware=middleware, **broker_options)
self.load_results_backend(broker, broker_settings)
dramatiq.set_broker(broker)
def execute():
"""Initialises MQ broker & connects dramatiq library.
"""
# Instantiate a broker.
broker = get_broker()
dramatiq.set_broker(broker)
# Inject middleware.
from stests.core.mq.middleware import get_middleware
for mware in get_middleware():
broker.add_middleware(mware)
# Inject encoder.
encoder.initialise()
dramatiq.set_encoder(encoder)
def initialize(cls):
global RATE_LIMITER_BACKEND
dramatiq.set_encoder(cls.select_encoder())
result_backend_settings = cls.result_backend_settings()
if result_backend_settings:
result_backend_path = result_backend_settings.get(
"BACKEND", "dramatiq.results.backends.StubBackend")
result_backend_class = import_string(result_backend_path)
result_backend_options = result_backend_settings.get(
"BACKEND_OPTIONS", {})
result_backend = result_backend_class(**result_backend_options)
results_middleware_options = result_backend_settings.get(
"MIDDLEWARE_OPTIONS", {})
results_middleware = Results(backend=result_backend,
**results_middleware_options)
else:
result_backend = None
results_middleware = None
rate_limiter_backend_settings = cls.rate_limiter_backend_settings()
if rate_limiter_backend_settings:
rate_limiter_backend_path = rate_limiter_backend_settings.get(
"BACKEND", "dramatiq.rate_limits.backends.stub.StubBackend")
rate_limiter_backend_class = import_string(
rate_limiter_backend_path)
rate_limiter_backend_options = rate_limiter_backend_settings.get(
"BACKEND_OPTIONS", {})
RATE_LIMITER_BACKEND = rate_limiter_backend_class(
**rate_limiter_backend_options)
broker_settings = cls.broker_settings()
broker_path = broker_settings["BROKER"]
broker_class = import_string(broker_path)
broker_options = broker_settings.get("OPTIONS", {})
middleware = [
load_middleware(path)
for path in broker_settings.get("MIDDLEWARE", [])
]
if result_backend is not None:
middleware.append(results_middleware)
broker = broker_class(middleware=middleware, **broker_options)
dramatiq.set_broker(broker)
def execute(mode: BrokerMode = BrokerMode.ACTORS):
"""Initialises dramatiq library.
:param mode: Mode in which MQ package is being used.
"""
# Instantiate a broker.
broker = get_broker()
dramatiq.set_broker(broker)
# Inject middleware.
from stests.core.mq.middleware import get_middleware
for mware in get_middleware(mode):
broker.add_middleware(mware)
# Simply broker & encoder.
dramatiq.set_encoder(encoder)
def execute():
"""Initialises MQ broker & connects dramatiq library.
"""
# JIT import to avoid circularity - TODO remove.
from stests.core.mq.middleware import get_middleware
# Configure broker.
broker = get_broker()
for mware in get_middleware():
broker.add_middleware(mware)
# Configure dramatiq.
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
log_event(EventType.CORE_BROKER_CONNECTION_ESTABLISHED, None)
def ready(self):
"""
Initialise our Broker when Django is ready.
"""
from . import signals # NOQA
encoder = self.get_encoder()
if encoder:
dramatiq.set_encoder(encoder())
options = self.get_broker_options()
middleware = self.get_middleware()
if middleware:
options["middleware"] = middleware
self.broker = PostgresBroker(**options)
dramatiq.set_broker(self.broker)
self.registry = self.get_registry()
self.registry.bind_broker(self.broker)
# dramatiq.set_encoder(dramatiq.PickleEncoder)
# result_backend = RedisBackend()
# middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus]
# broker = RabbitmqBroker(middleware=middleware)
# broker.add_middleware(Results(backend=result_backend))
# dramatiq.set_broker(broker)
import cv2
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.brokers.redis import RedisBroker
from dramatiq.middleware import default_middleware
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
result_backend = RedisBackend(encoder=dramatiq.PickleEncoder)
middleware = [
m() for m in default_middleware
if m is not dramatiq.middleware.prometheus.Prometheus
]
# broker = RabbitmqBroker(
# middleware=middleware
# )
broker = RedisBroker(middleware=middleware)
dramatiq.set_encoder(dramatiq.PickleEncoder)
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
import dramatiq
import requests
import sys
from dramatiq import group
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
@dramatiq.actor
def request(uri):
return requests.get(uri)
@dramatiq.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("uri", nargs="+", help="A website URI.")
def pickle_encoder():
old_encoder = dramatiq.get_encoder()
new_encoder = dramatiq.PickleEncoder()
dramatiq.set_encoder(new_encoder)
yield new_encoder
dramatiq.set_encoder(old_encoder)
class JSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
representation = obj.isoformat()
if representation.endswith("+00:00"):
representation = representation[:-6] + "Z"
return representation
return super(JSONEncoder, self).default(obj)
class DramatiqJSONEncoder(dramatiq.JSONEncoder):
def encode(self, data):
return json.dumps(data, separators=(",", ":"),
cls=JSONEncoder).encode("utf-8")
class DbConnectionsMiddleware(Middleware):
def _close_connections(self, *args, **kwargs):
db.connections.close_all()
before_consumer_thread_shutdown = _close_connections
before_worker_thread_shutdown = _close_connections
before_worker_shutdown = _close_connections
broker = RedisBroker(url=settings.BROKER_URL)
broker.add_middleware(DbConnectionsMiddleware())
dramatiq.set_broker(broker)
dramatiq.set_encoder(DramatiqJSONEncoder())
