def rabbitmq_broker(): broker = RabbitmqBroker(max_priority=10) check_rabbitmq(broker) broker.emit_after("process_boot") remoulade.set_broker(broker) yield broker broker.flush_all() broker.emit_before("process_stop") broker.close()
def rabbitmq_broker(): rmq_url = os.getenv("REMOULADE_TEST_RABBITMQ_URL") or "amqp://*****:*****@localhost:5784" broker = RabbitmqBroker(max_priority=10, url=rmq_url) check_rabbitmq(broker) broker.emit_after("process_boot") remoulade.set_broker(broker) yield broker broker.flush_all() broker.emit_before("process_stop") broker.close()
def test_can_instantiate_brokers_without_middleware(): # Given that I have an empty list of middleware # When I pass that to the RMQ Broker broker = RabbitmqBroker(middleware=[]) # Then I should get back a broker with not middleware assert not broker.middleware
from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.brokers.redis import RedisBroker logger = logging.getLogger("example") counter_key = "latench-bench-counter" memcache_client = pylibmc.Client(["localhost"], binary=True) memcache_pool = pylibmc.ClientPool(memcache_client, 8) random.seed(1337) if os.getenv("REDIS") == "1": broker = RedisBroker() remoulade.set_broker(broker) celery_app = celery.Celery(broker="redis:///") else: broker = RabbitmqBroker(host="127.0.0.1") remoulade.set_broker(broker) celery_app = celery.Celery(broker="amqp:///") def fib_bench(n): p, q = 0, 1 while n > 0: p, q = q, p + q n -= 1 with memcache_pool.reserve() as client: client.incr(counter_key) return p
import argparse import sys import remoulade import requests from remoulade import group from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.encoder import PickleEncoder from remoulade.results import Results from remoulade.results.backends import RedisBackend encoder = PickleEncoder() backend = RedisBackend(encoder=encoder) broker = RabbitmqBroker(host="127.0.0.1") broker.add_middleware(Results(backend=backend)) remoulade.set_broker(broker) remoulade.set_encoder(encoder) @remoulade.actor def request(uri): return requests.get(uri) @remoulade.actor(store_results=True) def count_words(response): return len(response.text.split(" ")) def main(): parser = argparse.ArgumentParser()
import random import sys import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker() remoulade.set_broker(broker) @remoulade.actor def add(x, y): add.logger.info("The sum of %d and %d is %d.", x, y, x + y) broker.declare_actor(add) def main(count): count = int(count) for _ in range(count): add.send(random.randint(0, 1000), random.randint(0, 1000)) if __name__ == "__main__": sys.exit(main(sys.argv[1]))
import remoulade from remoulade.scheduler import Scheduler, ScheduledJob from remoulade.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker(max_priority=10) remoulade.set_broker(broker) remoulade.set_scheduler( Scheduler(broker, [ ScheduledJob(actor_name="write_loaded_at", kwargs={ "filename": "/tmp/scheduler-2", "text": "simple schedule\n" }, interval=3600) ], period=0.1)) @remoulade.actor() def write_loaded_at(filename, text): with open(filename, "a+") as f: f.write(text) broker.declare_actor(write_loaded_at)
import os import random import time import pytest import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker() remoulade.set_broker(broker) @remoulade.actor(queue_name="benchmark-throughput") def throughput(): pass @remoulade.actor(queue_name="benchmark-fib") def fib(n): x, y = 1, 1 while n > 2: x, y = x + y, x n -= 1 return x @remoulade.actor(queue_name="benchmark-latency") def latency(): p = random.randint(1, 100) if p == 1:
from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.brokers.redis import RedisBroker logger = logging.getLogger("example") counter_key = "latench-bench-counter" memcache_client = pylibmc.Client(["localhost"], binary=True) memcache_pool = pylibmc.ClientPool(memcache_client, 8) random.seed(1337) if os.getenv("REDIS") == "1": broker = RedisBroker() remoulade.set_broker(broker) celery_app = celery.Celery(broker="redis:///") else: broker = RabbitmqBroker(url="127.0.0.1") remoulade.set_broker(broker) celery_app = celery.Celery(broker="amqp:///") def fib_bench(n): p, q = 0, 1 while n > 0: p, q = q, p + q n -= 1 with memcache_pool.reserve() as client: client.incr(counter_key) return p
import requests import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.scheduler import ScheduledJob, Scheduler broker = RabbitmqBroker() remoulade.set_broker(broker) @remoulade.actor() def count_words(url): response = requests.get(url).text print("There are {} words in {}".format(len(response), url)) broker.declare_actor(count_words) if __name__ == "__main__": scheduler = Scheduler( broker, [ ScheduledJob(actor_name="count_words", kwargs={"url": "https://github.com"}, interval=1), ScheduledJob(actor_name="count_words", kwargs={"url": "https://gitlab.com"}, interval=10), ], period=0.1, # scheduler run each 0.1 (second) )
import argparse import json import os import sys import time import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.middleware import Shutdown broker = RabbitmqBroker() remoulade.set_broker(broker) def path_to(*xs): return os.path.abspath( os.path.join(os.path.dirname(__file__), *(str(x) for x in xs))) def load_state(n): try: with open(path_to("states", n), "r") as f: data = json.load(f) except Exception: fib.logger.info("Could not read state file, using defaults.") return 1, 1, 0 i, x2, x1 = data["i"], data["x2"], data["x1"] fib.logger.info("Resuming fib(%d) from iteration %d.", n, i) return i, x2, x1
import argparse import random import sys import time import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.encoder import PickleEncoder from remoulade.results import Results from remoulade.results.backends import RedisBackend result_backend = RedisBackend(encoder=PickleEncoder()) broker = RabbitmqBroker() broker.add_middleware(Results(backend=result_backend)) remoulade.set_broker(broker) @remoulade.actor(store_results=True) def sleep_then_add(t, x, y): time.sleep(t) return x + y def main(args): parser = argparse.ArgumentParser() parser.add_argument("count", type=int, help="the number of messages to enqueue") args = parser.parse_args() messages = [] for _ in range(args.count): messages.append(sleep_then_add.send(random.randint(1, 5), random.randint(0, 1000), random.randint(0, 1000)))
import logging import re import sys from contextlib import closing from threading import local import requests import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker logger = logging.getLogger("example") anchor_re = re.compile(rb'<a href="([^"]+)">') state = local() broker = RabbitmqBroker() remoulade.set_broker(broker) urls = [] def get_session(): session = getattr(state, "session", None) if session is None: session = state.session = requests.Session() return session @remoulade.actor(max_retries=3, time_limit=10000) def crawl(url): if url in urls:
import random import sys import time import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker broker = RabbitmqBroker() remoulade.set_broker(broker) def fib(n): x, y = 1, 1 while n > 2: x, y = x + y, x n -= 1 return x @remoulade.actor(time_limit=86400000) def long_running(duration): deadline = time.monotonic() + duration while time.monotonic() < deadline: long_running.logger.info("%d seconds remaining.", deadline - time.monotonic()) n = random.randint(1000, 1000000) long_running.logger.info("Computing fib(%d).", n) fib(n) long_running.logger.info("Computed fib(%d).", n)
import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.scheduler import ScheduledJob, Scheduler broker = RabbitmqBroker(max_priority=10) remoulade.set_broker(broker) remoulade.set_scheduler( Scheduler( broker, [ ScheduledJob(actor_name="add", kwargs={ "x": 1, "y": 2 }, interval=1), ScheduledJob( actor_name="mul", kwargs={ "x": 1, "y": 2 }, interval=3600), ], period=0.1, )) @remoulade.actor() def add(x, y): return x + y @remoulade.actor() def mul(x, y):
import argparse import random import sys import time import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.encoder import PickleEncoder from remoulade.results import Results from remoulade.results.backends import RedisBackend result_backend = RedisBackend(encoder=PickleEncoder()) broker = RabbitmqBroker() broker.add_middleware(Results(backend=result_backend)) remoulade.set_broker(broker) @remoulade.actor(store_results=True) def sleep_then_add(t, x, y): time.sleep(t) return x + y broker.declare_actor(sleep_then_add) def main(): parser = argparse.ArgumentParser() parser.add_argument("count", type=int, help="the number of messages to enqueue") args = parser.parse_args()