def setup_dramatiq(config_class=Config): global broker, results_backend, rate_limiter_backend o = urlsplit(config_class.REDIS_URL) results_backend = RedisResultsBackend( host=o.hostname, port=o.port, db=o.path.split('/')[-1] or None, password=o.password ) rate_limiter_backend = RedisRateLimiterBackend( host=o.hostname, port=o.port, db=o.path.split('/')[-1] or None, password=o.password ) broker = RedisBroker(url=config_class.REDIS_URL) broker.add_middleware(Results(backend=results_backend)) dramatiq.set_broker(broker)
def setup_broker_and_backend(): redis_broker = RedisBroker(host="localhost", port=6379) result_backend = RedisBackend() dramatiq.set_broker(redis_broker) redis_broker.add_middleware(Results(backend=result_backend)) return redis_broker
# dramatiq.set_encoder(dramatiq.PickleEncoder) # result_backend = RedisBackend() # middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus] # broker = RabbitmqBroker(middleware=middleware) # broker.add_middleware(Results(backend=result_backend)) # dramatiq.set_broker(broker) import cv2 import dramatiq from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.brokers.redis import RedisBroker from dramatiq.middleware import default_middleware from dramatiq.results import Results from dramatiq.results.backends import RedisBackend result_backend = RedisBackend(encoder=dramatiq.PickleEncoder) middleware = [ m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus ] # broker = RabbitmqBroker( # middleware=middleware # ) broker = RedisBroker(middleware=middleware) dramatiq.set_encoder(dramatiq.PickleEncoder) broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(broker)
class JSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): representation = obj.isoformat() if representation.endswith("+00:00"): representation = representation[:-6] + "Z" return representation return super(JSONEncoder, self).default(obj) class DramatiqJSONEncoder(dramatiq.JSONEncoder): def encode(self, data): return json.dumps(data, separators=(",", ":"), cls=JSONEncoder).encode("utf-8") class DbConnectionsMiddleware(Middleware): def _close_connections(self, *args, **kwargs): db.connections.close_all() before_consumer_thread_shutdown = _close_connections before_worker_thread_shutdown = _close_connections before_worker_shutdown = _close_connections broker = RedisBroker(url=settings.BROKER_URL) broker.add_middleware(DbConnectionsMiddleware()) dramatiq.set_broker(broker) dramatiq.set_encoder(DramatiqJSONEncoder())
Run redis with `docker run --name dramatiq-redis -p 6379:6379 redis:5.0.4-alpine` Run as a worker with `dramatiq app` Run `ipython`, `import app`, and run e.g. `sleep.send(n)` """ import os import time import dramatiq from dramatiq.brokers.redis import RedisBroker from scout_apm.api import Config from scout_apm.dramatiq import ScoutMiddleware broker = RedisBroker() broker.add_middleware(ScoutMiddleware(), before=broker.middleware[0].__class__) dramatiq.set_broker(broker) Config.set( key=os.environ["SCOUT_KEY"], name="Test Dramatiq App", monitor=True, ) @dramatiq.actor(max_retries=0) def sleep(n): time.sleep(n) @dramatiq.actor(max_retries=0)
backend = RedisBackend(host=conf.REDIS_HOST, port=conf.REDIS_PORT, db=conf.BROKER_DB, password=conf.REDIS_PWD) DISTRIBUTED_MUTEX = ConcurrentRateLimiter(backend=backend, key='distributed-mutex', limit=st.CONCURRENT_LIMIT) redis_broker = RedisBroker(host=conf.REDIS_HOST, port=conf.REDIS_PORT, db=conf.BROKER_DB, password=conf.REDIS_PWD) STORE_RESULTS = False if conf.RESULT_BACKEND_DB: redis_broker.add_middleware( Results(backend=RedisBackend(host=conf.REDIS_HOST, port=conf.REDIS_PORT, db=conf.RESULT_BACKEND_DB, password=conf.REDIS_PWD))) STORE_RESULTS = True dramatiq.set_broker(redis_broker) def _crawler(task): """ 分布式调度的爬虫执行函数 :param task: 爬取任务 可以是str类型,如:http://www.example.com 可以是dict类型,遵循以下格式(其中url为必选): { "url": "http://www.example.com", "base_output_dir": "dir/to/save",
from dramatiq import Message from dramatiq.middleware import TimeLimitExceeded from dramatiq.results import Results from flask import Flask, render_template, redirect, request from dramatiq.brokers.redis import RedisBroker from dramatiq.results.backends import RedisBackend import dramatiq app = Flask(__name__) redis_broker = RedisBroker(host="localhost", port=6379) dramatiq_results = RedisBackend() redis_broker.add_middleware(Results(backend=dramatiq_results)) dramatiq.set_broker(redis_broker) log = dramatiq.logging.get_logger(__name__) TIME_LIMIT_ACTOR = 5000 values = [] message = [] @dramatiq.actor(store_results=True, time_limit=TIME_LIMIT_ACTOR) def prime_search(start_v, end_v): values = {} try: lower = start_v upper = end_v
context.push() self.state.context = context def after_process_message(self, broker, message, *, result=None, exception=None): try: context = self.state.context context.pop(exception) del self.state.context except AttributeError: pass after_skip_message = after_process_message broker.add_middleware(AppContextMiddleware(app)) @dramatiq.actor def parse_feeds(offset: int = 0): if offset == 0 and redis.get(FEED_PARSE_LOCK): logger.warning('Parsing already started') return redis.setex(FEED_PARSE_LOCK, FEED_PARSE_LOCK_TTL_SEC, 1) logger.info(f"Feed parsing started {offset}") feeds = select_feeds(offset, limit=FEED_PARSE_LIMIT) if not feeds:
from .config import instance as config from .db import Artifact, Content, db_session from .hasher import HashType, hash_file from .helpers import setup_logging from .log import instance as log from .watchers import get_watcher # brut.tasks is an entrypoint for workers, ensure logging is setup early setup_logging() # setup backend and broker for dramatiq actors prior to defining actors # must occur before the @dramatiq.actor decorator is used or you will never get task # messages being read from Redis redis_backend = RedisBackend() redis_broker = RedisBroker(url=config.redis) redis_broker.add_middleware(Results(backend=redis_backend)) dramatiq.set_broker(redis_broker) @dramatiq.actor def watch(watcher_type: str, *args, **kwargs): """Job responsible for getting new content entries and adding them to the db. Args: watcher_type (str): The type of watcher to use for extracting content.] """ watcher = get_watcher(watcher_type) if watcher is None:
from dramatiq.results.backends import RedisBackend from dramatiq.results import Results as DramatiqResults from dramatiq.actor import Actor from dramatiq import actor as dramatiq_actor from fzutils.ip_pools import tri_ip_pool from fzutils.spider.fz_requests import PROXY_TYPE_HTTPS from fzutils.spider.async_always import * redis_broker = RedisBroker( url='redis://127.0.0.1:6379/0',) result_backend = RedisBackend( url='redis://127.0.0.1:6379/1', ) dramatiq.set_broker(redis_broker) redis_broker.add_middleware( middleware=DramatiqResults( backend=result_backend)) @dramatiq_actor( broker=redis_broker, time_limit=1000 * 60, max_retries=None, max_age=1000 * 60 * 6, priority=0, # 默认值 store_results=True,) def get_url_body(url, use_proxy=False, proxy_type=PROXY_TYPE_HTTPS, num_retries=6,): body = Requests.get_url_body( url=url,
context.pop(exception) del self.state.context except AttributeError: pass after_skip_message = after_process_message app = Flask(__name__) api = Api(app) encoder = PickleEncoder() backend = RedisBackend(encoder=encoder) redis_broker = RedisBroker() redis_broker.add_middleware(FlaskMiddleware(app)) redis_broker.add_middleware(Results(backend=backend)) dramatiq.set_broker(redis_broker) @dramatiq.actor(store_results=True) def hello_task(): print("Hello, I'm Hello Task") return "ok" class HelloTask(Resource): def get(self): print("start run hello task") result = hello_task.send() print("Task Result: " + str(type(result)))