def ready(self): dramatiq.set_encoder(self.select_encoder()) result_backend_settings = self.result_backend_settings() if result_backend_settings: result_backend_path = result_backend_settings.get( "BACKEND", "dramatiq.results.backends.StubBackend") result_backend_class = load_class(result_backend_path) result_backend_options = result_backend_settings.get( "BACKEND_OPTIONS", {}) result_backend = result_backend_class(**result_backend_options) results_middleware_options = result_backend_settings.get( "MIDDLEWARE_OPTIONS", {}) results_middleware = Results(backend=result_backend, **results_middleware_options) else: result_backend = None results_middleware = None broker_settings = self.broker_settings() broker_path = broker_settings["BROKER"] broker_class = load_class(broker_path) broker_options = broker_settings.get("OPTIONS", {}) middleware = [ load_middleware(path) for path in broker_settings.get("MIDDLEWARE", []) ] if result_backend is not None: middleware.append(results_middleware) broker = broker_class(middleware=middleware, **broker_options) dramatiq.set_broker(broker)
def initialize_broker(broker, results_backend=None): dramatiq.set_broker(broker) dramatiq.set_encoder(middleware.TypedEncoder()) if results_backend: results_middleware = dramatiq.results.Results(backend=results_backend) broker.add_middleware(results_middleware) progress_middleware = middleware.ProgressMiddleware(cache.client) broker.add_middleware(progress_middleware)
def ready(self): broker_settings = self.broker_settings() broker_path = broker_settings["BROKER"] broker_class = load_class(broker_path) broker_options = broker_settings.get("OPTIONS", {}) middleware = [ load_middleware(path) for path in broker_settings.get("MIDDLEWARE", []) ] broker = broker_class(middleware=middleware, **broker_options) dramatiq.set_broker(broker) dramatiq.set_encoder(self.select_encoder())
def setup_redis_broker() -> None: _connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool( host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD ) _redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool) _result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db) _result_middleware = Results(backend=_result_backend) broker: Broker = RedisBroker( connection_pool=_connection_pool, middleware=[_result_middleware], namespace="lrw", ) dramatiq.set_broker(broker) dramatiq.set_encoder(dramatiq.PickleEncoder())
def ready(self): # this line should be first, to set up global encoder before any middleware is created dramatiq.set_encoder(self.select_encoder()) broker_settings = self.broker_settings() broker_path = broker_settings["BROKER"] broker_class = load_class(broker_path) broker_options = broker_settings.get("OPTIONS", {}) middleware = [ load_middleware(path) for path in broker_settings.get("MIDDLEWARE", []) ] broker = broker_class(middleware=middleware, **broker_options) self.load_results_backend(broker, broker_settings) dramatiq.set_broker(broker)
def execute(): """Initialises MQ broker & connects dramatiq library. """ # Instantiate a broker. broker = get_broker() dramatiq.set_broker(broker) # Inject middleware. from stests.core.mq.middleware import get_middleware for mware in get_middleware(): broker.add_middleware(mware) # Inject encoder. encoder.initialise() dramatiq.set_encoder(encoder)
def initialize(cls): global RATE_LIMITER_BACKEND dramatiq.set_encoder(cls.select_encoder()) result_backend_settings = cls.result_backend_settings() if result_backend_settings: result_backend_path = result_backend_settings.get( "BACKEND", "dramatiq.results.backends.StubBackend") result_backend_class = import_string(result_backend_path) result_backend_options = result_backend_settings.get( "BACKEND_OPTIONS", {}) result_backend = result_backend_class(**result_backend_options) results_middleware_options = result_backend_settings.get( "MIDDLEWARE_OPTIONS", {}) results_middleware = Results(backend=result_backend, **results_middleware_options) else: result_backend = None results_middleware = None rate_limiter_backend_settings = cls.rate_limiter_backend_settings() if rate_limiter_backend_settings: rate_limiter_backend_path = rate_limiter_backend_settings.get( "BACKEND", "dramatiq.rate_limits.backends.stub.StubBackend") rate_limiter_backend_class = import_string( rate_limiter_backend_path) rate_limiter_backend_options = rate_limiter_backend_settings.get( "BACKEND_OPTIONS", {}) RATE_LIMITER_BACKEND = rate_limiter_backend_class( **rate_limiter_backend_options) broker_settings = cls.broker_settings() broker_path = broker_settings["BROKER"] broker_class = import_string(broker_path) broker_options = broker_settings.get("OPTIONS", {}) middleware = [ load_middleware(path) for path in broker_settings.get("MIDDLEWARE", []) ] if result_backend is not None: middleware.append(results_middleware) broker = broker_class(middleware=middleware, **broker_options) dramatiq.set_broker(broker)
def execute(mode: BrokerMode = BrokerMode.ACTORS): """Initialises dramatiq library. :param mode: Mode in which MQ package is being used. """ # Instantiate a broker. broker = get_broker() dramatiq.set_broker(broker) # Inject middleware. from stests.core.mq.middleware import get_middleware for mware in get_middleware(mode): broker.add_middleware(mware) # Simply broker & encoder. dramatiq.set_encoder(encoder)
def execute(): """Initialises MQ broker & connects dramatiq library. """ # JIT import to avoid circularity - TODO remove. from stests.core.mq.middleware import get_middleware # Configure broker. broker = get_broker() for mware in get_middleware(): broker.add_middleware(mware) # Configure dramatiq. dramatiq.set_broker(broker) dramatiq.set_encoder(encoder) log_event(EventType.CORE_BROKER_CONNECTION_ESTABLISHED, None)
def ready(self): """ Initialise our Broker when Django is ready. """ from . import signals # NOQA encoder = self.get_encoder() if encoder: dramatiq.set_encoder(encoder()) options = self.get_broker_options() middleware = self.get_middleware() if middleware: options["middleware"] = middleware self.broker = PostgresBroker(**options) dramatiq.set_broker(self.broker) self.registry = self.get_registry() self.registry.bind_broker(self.broker)
# dramatiq.set_encoder(dramatiq.PickleEncoder) # result_backend = RedisBackend() # middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus] # broker = RabbitmqBroker(middleware=middleware) # broker.add_middleware(Results(backend=result_backend)) # dramatiq.set_broker(broker) import cv2 import dramatiq from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.brokers.redis import RedisBroker from dramatiq.middleware import default_middleware from dramatiq.results import Results from dramatiq.results.backends import RedisBackend result_backend = RedisBackend(encoder=dramatiq.PickleEncoder) middleware = [ m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus ] # broker = RabbitmqBroker( # middleware=middleware # ) broker = RedisBroker(middleware=middleware) dramatiq.set_encoder(dramatiq.PickleEncoder) broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(broker)
import dramatiq import requests import sys from dramatiq import group from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.encoder import PickleEncoder from dramatiq.results import Results from dramatiq.results.backends import RedisBackend encoder = PickleEncoder() backend = RedisBackend(encoder=encoder) broker = RabbitmqBroker() broker.add_middleware(Results(backend=backend)) dramatiq.set_broker(broker) dramatiq.set_encoder(encoder) @dramatiq.actor def request(uri): return requests.get(uri) @dramatiq.actor(store_results=True) def count_words(response): return len(response.text.split(" ")) def main(): parser = argparse.ArgumentParser() parser.add_argument("uri", nargs="+", help="A website URI.")
def pickle_encoder(): old_encoder = dramatiq.get_encoder() new_encoder = dramatiq.PickleEncoder() dramatiq.set_encoder(new_encoder) yield new_encoder dramatiq.set_encoder(old_encoder)
class JSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): representation = obj.isoformat() if representation.endswith("+00:00"): representation = representation[:-6] + "Z" return representation return super(JSONEncoder, self).default(obj) class DramatiqJSONEncoder(dramatiq.JSONEncoder): def encode(self, data): return json.dumps(data, separators=(",", ":"), cls=JSONEncoder).encode("utf-8") class DbConnectionsMiddleware(Middleware): def _close_connections(self, *args, **kwargs): db.connections.close_all() before_consumer_thread_shutdown = _close_connections before_worker_thread_shutdown = _close_connections before_worker_shutdown = _close_connections broker = RedisBroker(url=settings.BROKER_URL) broker.add_middleware(DbConnectionsMiddleware()) dramatiq.set_broker(broker) dramatiq.set_encoder(DramatiqJSONEncoder())