def stub_broker(r):
result_backend = RedisBackend()
result_backend.client = r
broker.add_middleware(Results(backend=result_backend))
broker.flush_all()
dramatiq.set_broker(stub_broker)
return broker
def redis_broker(r):
broker = RedisBroker()
broker.client = r
broker.client.flushall()
result_backend = RedisBackend()
result_backend.client = r
broker.add_middleware(Results(backend=result_backend))
broker.emit_after("process_boot")
# monkeypatch.setattr("yatsm.jobs.redis_broker", broker)
dramatiq.set_broker(broker)
yield broker
broker.client.flushall()
broker.close()
def configure():
result_backend = RedisBackend(port=63791)
rabbitmq_broker = RabbitmqBroker(port=56721)
rabbitmq_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(rabbitmq_broker)
def get_mware():
"""Returns a broker middleware designed to support making available actor execution results to downstream actors.
"""
return Results(
RedisBackend(client=redis.Redis(
db=EnvVars.DB, host=EnvVars.HOST, port=EnvVars.PORT)))
import re
import dramatiq
import sys
import geocoder
from utils import logger
from location import Location
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import JSONEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
result_backend = RedisBackend(encoder=JSONEncoder())
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
def _validate_input(location):
"""
Validate location input
@return True if address else false
"""
expr = r"[A-Za-z]+"
pattern = re.compile(expr)
match = pattern.findall(location)
if len(match) == 0:
# Provided input lat and long
return True
else:
import os
import requests
from . import config
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
from dramatiq.encoder import PickleEncoder
from dramatiq.brokers.redis import RedisBroker
from dramatiq import pipeline
from dramatiq import Middleware
import dramatiq
from threading import local
encoder = PickleEncoder()
backend = RedisBackend()
# encoder=encoder, host=config.HOST, port=config.PORT, db=config.DB, password=config.PASSWORD
broker = RedisBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
log = config.log
@dramatiq.actor(store_results=True)
def get_uri_contents(uri):
return requests.get(uri).text
# -*- coding:utf-8 -*-
import difflib
import requests
from bs4 import BeautifulSoup
import time
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
import redis
redis_ = redis.Redis(host='localhost',
port=6379,
charset="utf-8",
decode_responses=True)
result_backend = RedisBackend(url="redis://127.0.0.1:6379")
broker = RedisBroker(url="redis://127.0.0.1:6379")
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
def fill_db(ready_a, ready_d, job_id):
old_text = ready_a[0]
new_text = ready_d[0]
redis_.hset(job_id, 'old_text', old_text)
redis_.hset(job_id, 'new_text', new_text)
def realise_real_chunks(old_d, new_d, keys_to_realise_a, keys_to_realise_d,
real_spansa, real_spoansd, job_id):
full_filt_dic_o = {}
# dramatiq.set_encoder(dramatiq.PickleEncoder)
# result_backend = RedisBackend()
# middleware = [m() for m in default_middleware if m is not dramatiq.middleware.prometheus.Prometheus]
# broker = RabbitmqBroker(middleware=middleware)
# broker.add_middleware(Results(backend=result_backend))
# dramatiq.set_broker(broker)
import cv2
import dramatiq
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.brokers.redis import RedisBroker
from dramatiq.middleware import default_middleware
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
result_backend = RedisBackend(encoder=dramatiq.PickleEncoder)
middleware = [
m() for m in default_middleware
if m is not dramatiq.middleware.prometheus.Prometheus
]
# broker = RabbitmqBroker(
# middleware=middleware
# )
broker = RedisBroker(middleware=middleware)
dramatiq.set_encoder(dramatiq.PickleEncoder)
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
"""Batch processing with MapReduce, implemented with Dramatiq."""
import itertools
import dramatiq
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
from dramatiq.brokers.redis import RedisBroker
import requests
import spacy
# Dramatiq Setup
result_backend = RedisBackend(host="redis", port=6379)
redis_broker = RedisBroker(host="redis", port=6379)
redis_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(redis_broker)
# Using Spacy for entity extraction.
nlp = spacy.load("en_core_web_sm")
@dramatiq.actor(store_results=True)
def extract_ents(paragraph: str):
"""Extract entities from the paragraph."""
doc = nlp(paragraph)
ents = []
for ent in doc.ents:
if ent.label_ == "PERSON":
ents.append(ent.text)
from dramatiq import Message
from dramatiq.middleware import TimeLimitExceeded
from dramatiq.results import Results
from flask import Flask, render_template, redirect, request
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
import dramatiq
app = Flask(__name__)
redis_broker = RedisBroker(host="localhost", port=6379)
dramatiq_results = RedisBackend()
redis_broker.add_middleware(Results(backend=dramatiq_results))
dramatiq.set_broker(redis_broker)
log = dramatiq.logging.get_logger(__name__)
TIME_LIMIT_ACTOR = 5000
values = []
message = []
@dramatiq.actor(store_results=True, time_limit=TIME_LIMIT_ACTOR)
def prime_search(start_v, end_v):
values = {}
try:
lower = start_v
upper = end_v
import dramatiq import pika from dramatiq.brokers.rabbitmq import RabbitmqBroker from dramatiq.results import Results from dramatiq.results.backends import RedisBackend result_backend_connection = "redis://*****:*****@redis_backend:6379/0" broker_connection = "amqp://*****:*****@rabbit:5672" result_backend = RedisBackend(url=result_backend_connection) rabbitmq_broker = RabbitmqBroker(url=broker_connection) rabbitmq_broker.add_middleware(Results(backend=result_backend)) dramatiq.set_broker(rabbitmq_broker)
import argparse
import dramatiq
import random
import sys
import time
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
result_backend = RedisBackend()
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
@dramatiq.actor(store_results=True)
def sleep_then_add(t, x, y):
time.sleep(t)
return x + y
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("count",
type=int,
help="the number of messages to enqueue")
args = parser.parse_args()
messages = []
for _ in range(args.count):
import argparse
import dramatiq
import random
import sys
import time
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
result_backend = RedisBackend(encoder=PickleEncoder())
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(broker)
@dramatiq.actor(store_results=True)
def sleep_then_add(t, x, y):
time.sleep(t)
return x + y
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("count",
type=int,
help="the number of messages to enqueue")
args = parser.parse_args()
messages = []
from queue import Empty
from partridge.models import db
from dotenv import load_dotenv
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results
load_dotenv()
redis_broker = RedisBroker(host=os.environ.get("REDIS_HOST", 'localhost'),
password=os.environ.get("REDIS_PASSWORD", None),
namespace='partridge')
result_backend = RedisBackend(host=os.environ.get("REDIS_HOST", 'localhost'),
password=os.environ.get("REDIS_PASSWORD", None))
redis_broker.add_middleware(Results(backend=result_backend))
dramatiq.set_broker(redis_broker)
def get_minio_client() -> minio.Minio:
use_ssl = os.environ.get("MINIO_USE_SSL", "1") == '1'
mc = minio.Minio(os.environ.get("MINIO_HOST", "localhost:9000"),
access_key=os.environ.get("MINIO_ACCESS_KEY", ""),
secret_key=os.environ.get("MINIO_SECRET_KEY", ""),
secure=use_ssl)
"""
import dramatiq
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results.backends import RedisBackend
from dramatiq.results import Results as DramatiqResults
from dramatiq.actor import Actor
from dramatiq import actor as dramatiq_actor
from fzutils.ip_pools import tri_ip_pool
from fzutils.spider.fz_requests import PROXY_TYPE_HTTPS
from fzutils.spider.async_always import *
redis_broker = RedisBroker(
url='redis://127.0.0.1:6379/0',)
result_backend = RedisBackend(
url='redis://127.0.0.1:6379/1',
)
dramatiq.set_broker(redis_broker)
redis_broker.add_middleware(
middleware=DramatiqResults(
backend=result_backend))
@dramatiq_actor(
broker=redis_broker,
time_limit=1000 * 60,
max_retries=None,
max_age=1000 * 60 * 6,
priority=0, # 默认值
store_results=True,)
def get_url_body(url,
use_proxy=False,
import uuid
import os
from typing import List
from dacite import from_dict
from judger_error import JudgerError
from config import JUDGER_WORKSPACE, TEST_CASE_DIR
from compile_config import CompileConfig
from operation import Operation
from problem_info import ProblemInfo
from result import result
from run_config import RunConfig
from judger_task import Task
result_backend = RedisBackend(url="redis://container.ll-ap.cn:6379")
rabbitmq_broker = RabbitmqBroker(url="amqp://*****:*****@container.ll-ap.cn:5672")
rabbitmq_broker.add_middleware(Results(backend=result_backend))
# prometheus_middleware = Prometheus(http_host='0.0.0.0', http_port=9191)
# rabbitmq_broker.add_middleware(prometheus_middleware)
dramatiq.set_broker(rabbitmq_broker)
@dramatiq.actor(queue_name='judge', store_results=True)
def judge(task_id: str = None,
problem_id: str = None,
files: dict = None,
operations: List[str] = None,
config: dict = None):
if task_id is None or problem_id is None:
import argparse
import dramatiq
import requests
import sys
from dramatiq import group
from dramatiq.brokers.rabbitmq import RabbitmqBroker
from dramatiq.encoder import PickleEncoder
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=backend))
dramatiq.set_broker(broker)
dramatiq.set_encoder(encoder)
@dramatiq.actor
def request(uri):
return requests.get(uri)
@dramatiq.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
parser = argparse.ArgumentParser()
from dramatiq.results.backends import RedisBackend
from dramatiq.rate_limits import ConcurrentRateLimiter
import config as conf
from entrance import crawl_one_site
from logger import init_logger
from tasks import settings as st
init_logger(os.path.join(conf.LOG_DIR, 'page_collect.log'))
DISTRIBUTED_MUTEX = None
if st.CONCURRENT_LIMIT:
# 建立并发控制器
backend = RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
DISTRIBUTED_MUTEX = ConcurrentRateLimiter(backend=backend,
key='distributed-mutex',
limit=st.CONCURRENT_LIMIT)
redis_broker = RedisBroker(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.BROKER_DB,
password=conf.REDIS_PWD)
STORE_RESULTS = False
if conf.RESULT_BACKEND_DB:
redis_broker.add_middleware(
Results(backend=RedisBackend(host=conf.REDIS_HOST,
port=conf.REDIS_PORT,
db=conf.RESULT_BACKEND_DB,
password=conf.REDIS_PWD)))
def setup_broker_and_backend():
redis_broker = RedisBroker(host="localhost", port=6379)
result_backend = RedisBackend()
dramatiq.set_broker(redis_broker)
redis_broker.add_middleware(Results(backend=result_backend))
return redis_broker
import dramatiq
from random import randint
from time import sleep
from dramatiq.brokers.redis import RedisBroker
from dramatiq.results import Results
from dramatiq.results.backends import RedisBackend
broker = RedisBroker(host="redis")
dramatiq.set_broker(broker)
backend = RedisBackend(host="redis")
broker.add_middleware(Results(backend=backend))
@dramatiq.actor(store_results=True)
def heavy_calc(num: float):
sleep(randint(5, 50))
return num * 2