def rabbitmq_broker():
broker = RabbitmqBroker(max_priority=10)
check_rabbitmq(broker)
broker.emit_after("process_boot")
remoulade.set_broker(broker)
yield broker
broker.flush_all()
broker.emit_before("process_stop")
broker.close()
def rabbitmq_broker():
rmq_url = os.getenv("REMOULADE_TEST_RABBITMQ_URL") or "amqp://*****:*****@localhost:5784"
broker = RabbitmqBroker(max_priority=10, url=rmq_url)
check_rabbitmq(broker)
broker.emit_after("process_boot")
remoulade.set_broker(broker)
yield broker
broker.flush_all()
broker.emit_before("process_stop")
broker.close()
def test_can_instantiate_brokers_without_middleware():
# Given that I have an empty list of middleware
# When I pass that to the RMQ Broker
broker = RabbitmqBroker(middleware=[])
# Then I should get back a broker with not middleware
assert not broker.middleware
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.brokers.redis import RedisBroker
logger = logging.getLogger("example")
counter_key = "latench-bench-counter"
memcache_client = pylibmc.Client(["localhost"], binary=True)
memcache_pool = pylibmc.ClientPool(memcache_client, 8)
random.seed(1337)
if os.getenv("REDIS") == "1":
broker = RedisBroker()
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="redis:///")
else:
broker = RabbitmqBroker(host="127.0.0.1")
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="amqp:///")
def fib_bench(n):
p, q = 0, 1
while n > 0:
p, q = q, p + q
n -= 1
with memcache_pool.reserve() as client:
client.incr(counter_key)
return p
import argparse
import sys
import remoulade
import requests
from remoulade import group
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.encoder import PickleEncoder
from remoulade.results import Results
from remoulade.results.backends import RedisBackend
encoder = PickleEncoder()
backend = RedisBackend(encoder=encoder)
broker = RabbitmqBroker(host="127.0.0.1")
broker.add_middleware(Results(backend=backend))
remoulade.set_broker(broker)
remoulade.set_encoder(encoder)
@remoulade.actor
def request(uri):
return requests.get(uri)
@remoulade.actor(store_results=True)
def count_words(response):
return len(response.text.split(" "))
def main():
parser = argparse.ArgumentParser()
import random
import sys
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker()
remoulade.set_broker(broker)
@remoulade.actor
def add(x, y):
add.logger.info("The sum of %d and %d is %d.", x, y, x + y)
broker.declare_actor(add)
def main(count):
count = int(count)
for _ in range(count):
add.send(random.randint(0, 1000), random.randint(0, 1000))
if __name__ == "__main__":
sys.exit(main(sys.argv[1]))
import remoulade
from remoulade.scheduler import Scheduler, ScheduledJob
from remoulade.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker(max_priority=10)
remoulade.set_broker(broker)
remoulade.set_scheduler(
Scheduler(broker, [
ScheduledJob(actor_name="write_loaded_at",
kwargs={
"filename": "/tmp/scheduler-2",
"text": "simple schedule\n"
},
interval=3600)
],
period=0.1))
@remoulade.actor()
def write_loaded_at(filename, text):
with open(filename, "a+") as f:
f.write(text)
broker.declare_actor(write_loaded_at)
import os
import random
import time
import pytest
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker()
remoulade.set_broker(broker)
@remoulade.actor(queue_name="benchmark-throughput")
def throughput():
pass
@remoulade.actor(queue_name="benchmark-fib")
def fib(n):
x, y = 1, 1
while n > 2:
x, y = x + y, x
n -= 1
return x
@remoulade.actor(queue_name="benchmark-latency")
def latency():
p = random.randint(1, 100)
if p == 1:
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.brokers.redis import RedisBroker
logger = logging.getLogger("example")
counter_key = "latench-bench-counter"
memcache_client = pylibmc.Client(["localhost"], binary=True)
memcache_pool = pylibmc.ClientPool(memcache_client, 8)
random.seed(1337)
if os.getenv("REDIS") == "1":
broker = RedisBroker()
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="redis:///")
else:
broker = RabbitmqBroker(url="127.0.0.1")
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="amqp:///")
def fib_bench(n):
p, q = 0, 1
while n > 0:
p, q = q, p + q
n -= 1
with memcache_pool.reserve() as client:
client.incr(counter_key)
return p
import requests
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.scheduler import ScheduledJob, Scheduler
broker = RabbitmqBroker()
remoulade.set_broker(broker)
@remoulade.actor()
def count_words(url):
response = requests.get(url).text
print("There are {} words in {}".format(len(response), url))
broker.declare_actor(count_words)
if __name__ == "__main__":
scheduler = Scheduler(
broker,
[
ScheduledJob(actor_name="count_words",
kwargs={"url": "https://github.com"},
interval=1),
ScheduledJob(actor_name="count_words",
kwargs={"url": "https://gitlab.com"},
interval=10),
],
period=0.1, # scheduler run each 0.1 (second)
)
import argparse
import json
import os
import sys
import time
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.middleware import Shutdown
broker = RabbitmqBroker()
remoulade.set_broker(broker)
def path_to(*xs):
return os.path.abspath(
os.path.join(os.path.dirname(__file__), *(str(x) for x in xs)))
def load_state(n):
try:
with open(path_to("states", n), "r") as f:
data = json.load(f)
except Exception:
fib.logger.info("Could not read state file, using defaults.")
return 1, 1, 0
i, x2, x1 = data["i"], data["x2"], data["x1"]
fib.logger.info("Resuming fib(%d) from iteration %d.", n, i)
return i, x2, x1
import argparse
import random
import sys
import time
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.encoder import PickleEncoder
from remoulade.results import Results
from remoulade.results.backends import RedisBackend
result_backend = RedisBackend(encoder=PickleEncoder())
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=result_backend))
remoulade.set_broker(broker)
@remoulade.actor(store_results=True)
def sleep_then_add(t, x, y):
time.sleep(t)
return x + y
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument("count", type=int, help="the number of messages to enqueue")
args = parser.parse_args()
messages = []
for _ in range(args.count):
messages.append(sleep_then_add.send(random.randint(1, 5), random.randint(0, 1000), random.randint(0, 1000)))
import logging
import re
import sys
from contextlib import closing
from threading import local
import requests
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
logger = logging.getLogger("example")
anchor_re = re.compile(rb'<a href="([^"]+)">')
state = local()
broker = RabbitmqBroker()
remoulade.set_broker(broker)
urls = []
def get_session():
session = getattr(state, "session", None)
if session is None:
session = state.session = requests.Session()
return session
@remoulade.actor(max_retries=3, time_limit=10000)
def crawl(url):
if url in urls:
import random
import sys
import time
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
broker = RabbitmqBroker()
remoulade.set_broker(broker)
def fib(n):
x, y = 1, 1
while n > 2:
x, y = x + y, x
n -= 1
return x
@remoulade.actor(time_limit=86400000)
def long_running(duration):
deadline = time.monotonic() + duration
while time.monotonic() < deadline:
long_running.logger.info("%d seconds remaining.",
deadline - time.monotonic())
n = random.randint(1000, 1000000)
long_running.logger.info("Computing fib(%d).", n)
fib(n)
long_running.logger.info("Computed fib(%d).", n)
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.scheduler import ScheduledJob, Scheduler
broker = RabbitmqBroker(max_priority=10)
remoulade.set_broker(broker)
remoulade.set_scheduler(
Scheduler(
broker,
[
ScheduledJob(actor_name="add", kwargs={
"x": 1,
"y": 2
}, interval=1),
ScheduledJob(
actor_name="mul", kwargs={
"x": 1,
"y": 2
}, interval=3600),
],
period=0.1,
))
@remoulade.actor()
def add(x, y):
return x + y
@remoulade.actor()
def mul(x, y):
import argparse
import random
import sys
import time
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.encoder import PickleEncoder
from remoulade.results import Results
from remoulade.results.backends import RedisBackend
result_backend = RedisBackend(encoder=PickleEncoder())
broker = RabbitmqBroker()
broker.add_middleware(Results(backend=result_backend))
remoulade.set_broker(broker)
@remoulade.actor(store_results=True)
def sleep_then_add(t, x, y):
time.sleep(t)
return x + y
broker.declare_actor(sleep_then_add)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("count", type=int, help="the number of messages to enqueue")
args = parser.parse_args()
