def setup_dramatiq(config_class=Config):
global broker, results_backend, rate_limiter_backend
o = urlsplit(config_class.REDIS_URL)
results_backend = RedisResultsBackend(
host=o.hostname,
port=o.port,
db=o.path.split('/')[-1] or None,
password=o.password
)
rate_limiter_backend = RedisRateLimiterBackend(
host=o.hostname,
port=o.port,
db=o.path.split('/')[-1] or None,
password=o.password
)
broker = RedisBroker(url=config_class.REDIS_URL)
broker.add_middleware(Results(backend=results_backend))
dramatiq.set_broker(broker)
def setup_broker_and_backend():
redis_broker = RedisBroker(host="localhost", port=6379)
result_backend = RedisBackend()
dramatiq.set_broker(redis_broker)
redis_broker.add_middleware(Results(backend=result_backend))
return redis_broker
# dramatiq.set_encoder(dramatiq.PickleEncoder)
# result_backend = RedisBackend()
# middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus]
# broker = RabbitmqBroker(middleware=middleware)
# broker.add_middleware(Results(backend=result_backend))
# dramatiq.set_broker(broker)
import cv2
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.brokers.redis import RedisBroker
from dramatiq.middleware import default_middleware
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
result_backend = RedisBackend(encoder=dramatiq.PickleEncoder)
middleware = [
m() for m in default_middleware
if m is not dramatiq.middleware.prometheus.Prometheus
]
# broker = RabbitmqBroker(
# middleware=middleware
# )
broker = RedisBroker(middleware=middleware)
dramatiq.set_encoder(dramatiq.PickleEncoder)
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
class JSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
representation = obj.isoformat()
if representation.endswith("+00:00"):
representation = representation[:-6] + "Z"
return representation
return super(JSONEncoder, self).default(obj)
class DramatiqJSONEncoder(dramatiq.JSONEncoder):
def encode(self, data):
return json.dumps(data, separators=(",", ":"),
cls=JSONEncoder).encode("utf-8")
class DbConnectionsMiddleware(Middleware):
def _close_connections(self, *args, **kwargs):
db.connections.close_all()
before_consumer_thread_shutdown = _close_connections
before_worker_thread_shutdown = _close_connections
before_worker_shutdown = _close_connections
broker = RedisBroker(url=settings.BROKER_URL)
broker.add_middleware(DbConnectionsMiddleware())
dramatiq.set_broker(broker)
dramatiq.set_encoder(DramatiqJSONEncoder())
Run redis with `docker run --name dramatiq-redis -p 6379:6379 redis:5.0.4-alpine`
Run as a worker with `dramatiq app`
Run `ipython`, `import app`, and run e.g. `sleep.send(n)`
"""
import os
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from scout_apm.api import Config
from scout_apm.dramatiq import ScoutMiddleware
broker = RedisBroker()
broker.add_middleware(ScoutMiddleware(), before=broker.middleware[0].__class__)
dramatiq.set_broker(broker)
Config.set(
key=os.environ["SCOUT_KEY"],
name="Test Dramatiq App",
monitor=True,
)
@dramatiq.actor(max_retries=0)
def sleep(n):
time.sleep(n)
@dramatiq.actor(max_retries=0)
backend = RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
DISTRIBUTED_MUTEX = ConcurrentRateLimiter(backend=backend,
key='distributed-mutex',
limit=st.CONCURRENT_LIMIT)
redis_broker = RedisBroker(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
STORE_RESULTS = False
if conf.RESULT_BACKEND_DB:
redis_broker.add_middleware(
Results(backend=RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.RESULT_BACKEND_DB,
password=conf.REDIS_PWD)))
STORE_RESULTS = True
dramatiq.set_broker(redis_broker)
def _crawler(task):
"""
分布式调度的爬虫执行函数
:param task: 爬取任务
可以是str类型，如：http://www.example.com
可以是dict类型，遵循以下格式（其中url为必选）：
{
"url": "http://www.example.com",
"base_output_dir": "dir/to/save",
from dramatiq import Message
from dramatiq.middleware import TimeLimitExceeded
from dramatiq.results import Results
from flask import Flask, render_template, redirect, request
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
import dramatiq
app = Flask(__name__)
redis_broker = RedisBroker(host="localhost", port=6379)
dramatiq_results = RedisBackend()
redis_broker.add_middleware(Results(backend=dramatiq_results))
dramatiq.set_broker(redis_broker)
log = dramatiq.logging.get_logger(__name__)
TIME_LIMIT_ACTOR = 5000
values = []
message = []
@dramatiq.actor(store_results=True, time_limit=TIME_LIMIT_ACTOR)
def prime_search(start_v, end_v):
values = {}
try:
lower = start_v
upper = end_v
context.push()
self.state.context = context
def after_process_message(self, broker, message, *, result=None, exception=None):
try:
context = self.state.context
context.pop(exception)
del self.state.context
except AttributeError:
pass
after_skip_message = after_process_message
broker.add_middleware(AppContextMiddleware(app))
@dramatiq.actor
def parse_feeds(offset: int = 0):
if offset == 0 and redis.get(FEED_PARSE_LOCK):
logger.warning('Parsing already started')
return
redis.setex(FEED_PARSE_LOCK, FEED_PARSE_LOCK_TTL_SEC, 1)
logger.info(f"Feed parsing started {offset}")
feeds = select_feeds(offset, limit=FEED_PARSE_LIMIT)
if not feeds:
from .config import instance as config
from .db import Artifact, Content, db_session
from .hasher import HashType, hash_file
from .helpers import setup_logging
from .log import instance as log
from .watchers import get_watcher
# brut.tasks is an entrypoint for workers, ensure logging is setup early
setup_logging()
# setup backend and broker for dramatiq actors prior to defining actors
# must occur before the @dramatiq.actor decorator is used or you will never get task
# messages being read from Redis
redis_backend = RedisBackend()
redis_broker = RedisBroker(url=config.redis)
redis_broker.add_middleware(Results(backend=redis_backend))
dramatiq.set_broker(redis_broker)
@dramatiq.actor
def watch(watcher_type: str, *args, **kwargs):
"""Job responsible for getting new content entries and adding them to the db.
Args:
watcher_type (str):
The type of watcher to use for extracting content.]
"""
watcher = get_watcher(watcher_type)
if watcher is None:
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results as DramatiqResults
from dramatiq.actor import Actor
from dramatiq import actor as dramatiq_actor
from fzutils.ip_pools import tri_ip_pool
from fzutils.spider.fz_requests import PROXY_TYPE_HTTPS
from fzutils.spider.async_always import *
redis_broker = RedisBroker(
url='redis://127.0.0.1:6379/0',)
result_backend = RedisBackend(
url='redis://127.0.0.1:6379/1',
)
dramatiq.set_broker(redis_broker)
redis_broker.add_middleware(
middleware=DramatiqResults(
backend=result_backend))
@dramatiq_actor(
broker=redis_broker,
time_limit=1000 * 60,
max_retries=None,
max_age=1000 * 60 * 6,
priority=0, # 默认值
store_results=True,)
def get_url_body(url,
use_proxy=False,
proxy_type=PROXY_TYPE_HTTPS,
num_retries=6,):
body = Requests.get_url_body(
url=url,
context.pop(exception)
del self.state.context
except AttributeError:
pass
after_skip_message = after_process_message
app = Flask(__name__)
api = Api(app)
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
redis_broker = RedisBroker()
redis_broker.add_middleware(FlaskMiddleware(app))
redis_broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(redis_broker)
@dramatiq.actor(store_results=True)
def hello_task():
print("Hello, I'm Hello Task")
return "ok"
class HelloTask(Resource):
def get(self):
print("start run hello task")
result = hello_task.send()
print("Task Result: " + str(type(result)))