def redis_broker(r):
broker = RedisBroker()
broker.client = r
broker.client.flushall()
result_backend = RedisBackend()
result_backend.client = r
broker.add_middleware(Results(backend=result_backend))
broker.emit_after("process_boot")
# monkeypatch.setattr("yatsm.jobs.redis_broker", broker)
dramatiq.set_broker(broker)
yield broker
broker.client.flushall()
broker.close()
def test_redis_broker_can_connect_via_url():
# Given that I have a connection string
# When I pass that to RedisBroker
broker = RedisBroker(url="redis://127.0.0.1")
# Then I should get back a valid connection
assert broker.client.ping()
def test_redis_broker_warns_about_deprecated_parameters():
# When I pass deprecated params to RedisBroker
# Then it should warn me that those params do nothing
with pytest.warns(DeprecationWarning) as record:
RedisBroker(requeue_deadline=1000)
assert str(record[0].message) == \
"requeue_{deadline,interval} have been deprecated and no longer do anything"
def get_broker() -> RedisBroker:
"""Returns instance of redis MQ broker.
"""
return RedisBroker(db=EnvVars.DB,
host=EnvVars.HOST,
port=EnvVars.PORT,
namespace="stests")
def setup_broker(settings):
"""Setup all brokers."""
from dramatiq.brokers.redis import RedisBroker
dramatiq.set_broker(RedisBroker(url=settings['app.broker.url']))
dramatiq.get_broker().add_middleware(ConfigMiddleware())
dramatiq.get_broker().add_middleware(DBSessionMiddleware())
dramatiq.get_broker().add_middleware(ElasticsearchMiddleware())
dramatiq.get_broker().add_middleware(RateLimiterMiddleware())
def test_redis_broker_can_connect_via_client():
# Given that I already have redis client in hand
# When I pass it to RedisBroker
client = redis.Redis()
broker = RedisBroker(client=client)
# Then I should get back a valid connection
assert broker.client.ping()
assert broker.client is client
def initialize_task_worker(
dramatiq_processes: int = 1,
dramatiq_log_file: Optional[str] = None,
):
# All processes in the current process group will be terminated
# with the lead process.
try:
os.setpgrp()
except PermissionError:
# TODO: Occurred in github action. Investigate the root cause.
pass
atexit.register(stop_all_children_processes)
# Run Redis.
redis_port = random_select_port()
pgid = os.getpgrp()
subprocess.Popen(
[redis_server.REDIS_SERVER_PATH, '--port', redis_port],
# Attach to the current process group.
preexec_fn=lambda: os.setpgid(0, pgid),
# Suppress stdout and stderr.
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
# Set broker.
from dramatiq.brokers.redis import RedisBroker
dynamic_dramatiq.set_broker(RedisBroker(host='localhost', port=redis_port))
# Run worker.
dramatiq_env = dict(os.environ)
dramatiq_env['DYNAMIC_DRAMATIQ_REDIS_BROKER_PORT'] = redis_port
dramatiq_command = [
'dramatiq',
'pywharf_core.job',
'--processes',
str(dramatiq_processes),
]
if dramatiq_log_file:
dramatiq_command.extend([
'--log-file',
dramatiq_log_file,
])
subprocess.Popen(
dramatiq_command,
# Share env.
env=dramatiq_env,
# Attach to the current process group.
preexec_fn=lambda: os.setpgid(0, pgid),
# Suppress stdout and stderr.
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def setup_worker():
"""Set up Dramatiq with Redis as a queue.
Dramatiq manages the message passing to background workers which run
long tasks to avoid stalling the application responses for too long.
Redis is used as a message queue for simplicity sake. A more robust
infrastructure could configure a RabbitMQ here, for example.
"""
logger.info('creating tasks queue')
broker = RedisBroker(url=f'{settings.REDIS_URL}')
dramatiq.set_broker(broker)
def setup_dramatiq(config_class=Config):
global broker, results_backend, rate_limiter_backend
o = urlsplit(config_class.REDIS_URL)
results_backend = RedisResultsBackend(
host=o.hostname,
port=o.port,
db=o.path.split('/')[-1] or None,
password=o.password
)
rate_limiter_backend = RedisRateLimiterBackend(
host=o.hostname,
port=o.port,
db=o.path.split('/')[-1] or None,
password=o.password
)
broker = RedisBroker(url=config_class.REDIS_URL)
broker.add_middleware(Results(backend=results_backend))
dramatiq.set_broker(broker)
def redis_broker():
broker = RedisBroker()
broker.client.flushall()
broker.emit_after("process_boot")
dramatiq.set_broker(broker)
yield broker
broker.client.flushall()
broker.close()
def setup_redis_broker() -> None:
_connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool(
host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD
)
_redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool)
_result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db)
_result_middleware = Results(backend=_result_backend)
broker: Broker = RedisBroker(
connection_pool=_connection_pool,
middleware=[_result_middleware],
namespace="lrw",
)
dramatiq.set_broker(broker)
dramatiq.set_encoder(dramatiq.PickleEncoder())
def get_dramatiq_broker_object(self):
"""
This method initializes the broker object for Dramatiq and saves it in
Django's settings.
"""
if self.DRAMATIQ_BROKER_URL:
url = self.DRAMATIQ_BROKER_URL.strip()
if url is None:
kind, host, port = "stub", None, None
else:
kind, _, url = url.partition("://")
host, _, port = url.partition(":")
host = host or None
port = int(port) if port else None
else:
kind = self.DRAMATIQ_BROKER_TYPE
host = self.DRAMATIQ_BROKER_HOST or None
port = self.DRAMATIQ_BROKER_PORT or None
# Separate non-null args
kwargs = [("host", host), ("port", port)]
kwargs = {k: v for k, v in kwargs if v is not None}
# Initializes broker
if kind == "stub":
from dramatiq.brokers.stub import StubBroker
broker = StubBroker()
elif kind == "redis":
from dramatiq.brokers.redis import RedisBroker
broker = RedisBroker(**kwargs)
elif kind == "rabbitmq":
from dramatiq.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(**kwargs)
else:
raise ValueError(f"invalid dramatiq broker: {kind}")
# Configure as default and exit
dramatiq.set_broker(broker)
return broker
# dramatiq.set_encoder(dramatiq.PickleEncoder)
# result_backend = RedisBackend()
# middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus]
# broker = RabbitmqBroker(middleware=middleware)
# broker.add_middleware(Results(backend=result_backend))
# dramatiq.set_broker(broker)
import cv2
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.brokers.redis import RedisBroker
from dramatiq.middleware import default_middleware
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
result_backend = RedisBackend(encoder=dramatiq.PickleEncoder)
middleware = [
m() for m in default_middleware
if m is not dramatiq.middleware.prometheus.Prometheus
]
# broker = RabbitmqBroker(
# middleware=middleware
# )
broker = RedisBroker(middleware=middleware)
dramatiq.set_encoder(dramatiq.PickleEncoder)
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
import dramatiq from dramatiq.brokers.redis import RedisBroker import settings broker = RedisBroker(url=settings.REDIS_URL) dramatiq.set_broker(broker)
from .base import *
from .contrib import *
from .utils import ABS_PATH
from hdx.hdx_configuration import Configuration
# Project apps
INSTALLED_APPS += (
'jobs',
'tasks',
'api',
'ui',
'utils',
)
dramatiq.set_broker(RedisBroker(host="localhost", port=6379))
DATABASES = {}
DATABASES['default'] = dj_database_url.config(default='postgis:///exports',
conn_max_age=500)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': ['api/templates/', 'ui/templates', 'ui/static/ui/js'],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
import os
import json
import shutil
import dramatiq
import requests
from zipfile import ZipFile
from dramatiq.brokers.redis import RedisBroker
from app.configure import appConfig
if 'REDIS_HOST' in os.environ:
redis_broker = RedisBroker(url=os.environ['REDIS_HOST'])
else:
redis_broker = RedisBroker(host="redis", port=6379)
dramatiq.set_broker(redis_broker)
@dramatiq.actor()
def createDump(batchPath, setPath, batchTag, sourceNamespace, setNamespace,
baseURI):
file_paths = []
for root, directories, files in os.walk(batchPath):
for filename in files:
# join the two strings in order to form the full filepath.
filepath = os.path.join(root, filename)
file_paths.append(filepath)
zipPath = str(setPath) + "/" + str(batchTag) + ".zip"
with ZipFile(zipPath, 'w') as zip:
for file in file_paths:
arcPath = file.replace(setPath, '')
"""Batch processing with MapReduce, implemented with Dramatiq."""
import itertools
import dramatiq
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
from dramatiq.brokers.redis import RedisBroker
import requests
import spacy
# Dramatiq Setup
result_backend = RedisBackend(host="redis", port=6379)
redis_broker = RedisBroker(host="redis", port=6379)
redis_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(redis_broker)
# Using Spacy for entity extraction.
nlp = spacy.load("en_core_web_sm")
@dramatiq.actor(store_results=True)
def extract_ents(paragraph: str):
"""Extract entities from the paragraph."""
doc = nlp(paragraph)
ents = []
for ent in doc.ents:
if ent.label_ == "PERSON":
ents.append(ent.text)
import requests
import glob
import sentry_sdk
import sentry_dramatiq
from time import sleep
from threading import Thread
if __name__ == "__main__":
if hasattr(tasks_config, 'SENTRY_DSN'):
sentry_sdk.init(
tasks_config.SENTRY_DSN,
integrations=[sentry_dramatiq.DramatiqIntegration()],
)
redis_broker = RedisBroker(url=tasks_config.REDIS,
middleware=[],
namespace=tasks_config.DRAMATIQ_NS)
dramatiq.set_broker(redis_broker)
RECORD_WAV_CFG = """
timescale 1
com_maxfps 125
timescale 1
echo "^1oooooooooooooooooo ^5WAV_RECORD ^1ooooooooooooooooooooooooooo"
//exec cameras\\runs\\1stPos
wav_record synctest
set nextdemo quit
"""
PREINIT_WAV_CFG = """
cl_avidemo 0
# -*- coding: utf-8 -*-
import dramatiq
import time
from elasticsearch import Elasticsearch
from dramatiq.brokers.redis import RedisBroker
# from dramatiq.results.backends import RedisBackend
# from dramatiq.results import Results
import config as conf
# result_backend = RedisBackend(url="redis://127.0.0.1:16379/14")
redis_broker = RedisBroker(url="redis://127.0.0.1:16379/15")
# redis_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(redis_broker)
es = Elasticsearch(hosts=conf.ES_HOST)
@dramatiq.actor(queue_name='log')
def log_task(question, answer, target):
body = {
"question": question,
"answer": answer,
"target": target,
"createdAt": int(time.time())
}
es.index(index='fo-log-index', doc_type='chat', body=body)
def setup_broker():
redis_broker = RedisBroker(host="localhost", port=6379)
dramatiq.set_broker(redis_broker)
return redis_broker
from dramatiq.brokers.redis import RedisBroker
from .configurations import get_configurations
from .external.pontomais import PontoMais
import dramatiq
CONFIGURATIONS = get_configurations()
redis_broker = RedisBroker(host=CONFIGURATIONS.get("message_broker",
"endpoint"),
port=CONFIGURATIONS.getint("message_broker",
"port"),
db=CONFIGURATIONS.getint("message_broker",
"database"))
dramatiq.set_broker(redis_broker)
@dramatiq.actor
def register_punch(email: str, password: str, address: str, latitude: float,
longitude: float):
pontomais_api = PontoMais(user_login=email, user_password=password)
pontomais_api.register_punch(address=address,
latitude=latitude,
longitude=longitude)
import os
import dramatiq
from kubernetes import client, config
from __init__ import create_app
from models import db, Workload
brokerService = os.getenv("DRAMATIQ_BROKER")
config.load_incluster_config()
apps_v1 = client.AppsV1Api()
app = create_app()
if brokerService.startswith('redis'):
from dramatiq.brokers.redis import RedisBroker
redis_broker = RedisBroker(url=brokerService)
dramatiq.set_broker(redis_broker)
else:
from dramatiq.brokers.rabbitmq import RabbitmqBroker
rabbitmq_broker = RabbitmqBroker(url=brokerService)
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor
def stop_workload(id, workload, workload_type, namespace):
if workload_type == 'ReplicaSet':
resp = apps_v1.list_namespaced_deployment(namespace=namespace)
for i in resp.items:
if i.metadata.name == workload:
i.spec.replicas = 0
apps_v1.patch_namespaced_deployment(name=workload,
from . import config
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
from dramatiq.encoder import PickleEncoder
from dramatiq.brokers.redis import RedisBroker
from dramatiq import pipeline
from dramatiq import Middleware
import dramatiq
from threading import local
encoder = PickleEncoder()
backend = RedisBackend()
# encoder=encoder, host=config.HOST, port=config.PORT, db=config.DB, password=config.PASSWORD
broker = RedisBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
log = config.log
@dramatiq.actor(store_results=True)
def get_uri_contents(uri):
return requests.get(uri).text
@dramatiq.actor(store_results=True)
def count_words(uri, text):
import argparse
import dramatiq
import os
import random
import sys
if os.getenv("REDIS") == "1":
from dramatiq.brokers.redis import RedisBroker
broker = RedisBroker()
dramatiq.set_broker(broker)
@dramatiq.actor
def add(x, y):
add.logger.info("The sum of %d and %d is %d.", x, y, x + y)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("count", type=int, help="the number of messages to enqueue")
args = parser.parse_args()
for _ in range(args.count):
add.send(random.randint(0, 1000), random.randint(0, 1000))
if __name__ == "__main__":
sys.exit(main(sys.argv))
from tasks import settings as st
init_logger(os.path.join(conf.LOG_DIR, 'page_collect.log'))
DISTRIBUTED_MUTEX = None
if st.CONCURRENT_LIMIT:
# 建立并发控制器
backend = RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
DISTRIBUTED_MUTEX = ConcurrentRateLimiter(backend=backend,
key='distributed-mutex',
limit=st.CONCURRENT_LIMIT)
redis_broker = RedisBroker(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
STORE_RESULTS = False
if conf.RESULT_BACKEND_DB:
redis_broker.add_middleware(
Results(backend=RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.RESULT_BACKEND_DB,
password=conf.REDIS_PWD)))
STORE_RESULTS = True
dramatiq.set_broker(redis_broker)
def _crawler(task):
"""
分布式调度的爬虫执行函数
class InitDB(dramatiq.Middleware):
def before_worker_boot(self, broker, worker):
async def run():
from . import db
await db.db.set_bind(db.CONNECTION_STR)
loop.run_until_complete(run())
def before_worker_shutdown(self, broker, worker):
shutdown.set()
MIDDLEWARE = [
m()
for m in (AgeLimit, ShutdownNotifications, Callbacks, Pipelines, Retries)
]
if TEST:
broker = StubBroker(middleware=MIDDLEWARE)
broker.emit_after("process_boot")
else:
broker = RedisBroker(
connection_pool=redis.ConnectionPool.from_url(REDIS_HOST),
middleware=MIDDLEWARE)
broker.add_middleware(InitDB())
dramatiq.set_broker(broker)
class JSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
representation = obj.isoformat()
if representation.endswith("+00:00"):
representation = representation[:-6] + "Z"
return representation
return super(JSONEncoder, self).default(obj)
class DramatiqJSONEncoder(dramatiq.JSONEncoder):
def encode(self, data):
return json.dumps(data, separators=(",", ":"),
cls=JSONEncoder).encode("utf-8")
class DbConnectionsMiddleware(Middleware):
def _close_connections(self, *args, **kwargs):
db.connections.close_all()
before_consumer_thread_shutdown = _close_connections
before_worker_thread_shutdown = _close_connections
before_worker_shutdown = _close_connections
broker = RedisBroker(url=settings.BROKER_URL)
broker.add_middleware(DbConnectionsMiddleware())
dramatiq.set_broker(broker)
dramatiq.set_encoder(DramatiqJSONEncoder())
"""
Run redis with `docker run --name dramatiq-redis -p 6379:6379 redis:5.0.4-alpine`
Run as a worker with `dramatiq app`
Run `ipython`, `import app`, and run e.g. `sleep.send(n)`
"""
import os
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from scout_apm.api import Config
from scout_apm.dramatiq import ScoutMiddleware
broker = RedisBroker()
broker.add_middleware(ScoutMiddleware(), before=broker.middleware[0].__class__)
dramatiq.set_broker(broker)
Config.set(
key=os.environ["SCOUT_KEY"],
name="Test Dramatiq App",
monitor=True,
)
@dramatiq.actor(max_retries=0)
def sleep(n):
time.sleep(n)
def setup_broker_and_backend():
redis_broker = RedisBroker(host="localhost", port=6379)
result_backend = RedisBackend()
dramatiq.set_broker(redis_broker)
redis_broker.add_middleware(Results(backend=result_backend))
return redis_broker
from dramatiq import Message
from dramatiq.middleware import TimeLimitExceeded
from dramatiq.results import Results
from flask import Flask, render_template, redirect, request
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
import dramatiq
app = Flask(__name__)
redis_broker = RedisBroker(host="localhost", port=6379)
dramatiq_results = RedisBackend()
redis_broker.add_middleware(Results(backend=dramatiq_results))
dramatiq.set_broker(redis_broker)
log = dramatiq.logging.get_logger(__name__)
TIME_LIMIT_ACTOR = 5000
values = []
message = []
@dramatiq.actor(store_results=True, time_limit=TIME_LIMIT_ACTOR)
def prime_search(start_v, end_v):
values = {}
try:
lower = start_v
upper = end_v
