def run_feed_indexer(context):
try:
worker_pool = Pool()
@pooled(worker_pool)
@always_ack
def cb(message_data, message):
try:
with context:
return handle_message(message_data, message, context)
except GreenletExit:
pass
except:
log.error("Unexpected error handling feed indexer message: %s" % traceback.format_exc())
with context:
dispatch = MessageDispatch(context)
proc = dispatch.start_worker(INDEX_FEED_COMMAND, cb)
proc.wait()
except GreenletExit:
pass
except:
log.error("Unexpected error running feed indexer: %s" % traceback.format_exc())
finally:
# stop accepting work
proc.kill()
proc.wait()
# stop working on existing work
worker_pool.killall()
worker_pool.waitall()
def test_dispatch_one_receiver(ctx):
from eventlet import sleep
from eventlet.event import Event
from melkman.messaging import MessageDispatch, always_ack
w = MessageDispatch(ctx)
message_type = "test_dispatch_one_receiver"
work_result = Event()
got_events = {"count": 0}
@always_ack
def handler(job, message):
got_events["count"] += 1
worker1 = w.start_worker(message_type, handler)
worker2 = w.start_worker(message_type, handler)
try:
w.send({}, message_type)
sleep(2)
assert got_events["count"] == 1
finally:
worker1.kill()
worker1.wait()
worker2.kill()
worker2.wait()
def test_defer_message_dispatch(ctx):
from datetime import datetime, timedelta
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from melkman.messaging import MessageDispatch, always_ack
from melkman.scheduler import defer_message
from melkman.scheduler.worker import ScheduledMessageService
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
w = MessageDispatch(ctx)
message_type = 'test_dispatch_send_recv'
work_result = Event()
@always_ack
def handler(job, message):
work_result.send(sum(job['values']))
worker = w.start_worker(message_type, handler)
try:
now = datetime.utcnow()
wait = timedelta(seconds=2)
# w.send({'values': [1, 2]}, message_type)
defer_message(now + wait, {'values': [1 ,2]}, message_type, ctx)
sleep(3)
assert with_timeout(2, work_result.wait) == 3
finally:
worker.kill()
worker.wait()
sched.kill()
sched.wait()
def run_aggregator(context):
try:
worker_pool = Pool()
@pooled(worker_pool)
@always_ack
def bucket_modified_handler(message_data, message):
try:
with context:
_handle_bucket_modified(message_data, message, context)
except GreenletExit:
pass
except:
log.error("Unexpected error handling bucking modified message: %s" % traceback.format_exc())
@pooled(worker_pool)
@always_ack
def update_subscription_handler(message_data, message):
try:
with context:
_handle_update_subscription(message_data, message, context)
except GreenletExit:
pass
except:
log.error("Unexpected error handling update subscription message: %s" % traceback.format_exc())
procs = []
with context:
dispatcher = MessageDispatch(context)
procs.append(dispatcher.start_worker(BUCKET_MODIFIED, bucket_modified_handler))
procs.append(dispatcher.start_worker(UPDATE_SUBSCRIPTION, update_subscription_handler))
waitall(procs)
except GreenletExit:
pass
except:
log.error("Unexpected error running aggregator process: %s" % traceback.format_exc())
finally:
# stop accepting work
killall(procs)
waitall(procs)
# stop working on existing work
worker_pool.killall()
worker_pool.waitall()
def _start_listener(self):
@always_ack
def cb(message_data, message):
with self.context:
_handle_scheduler_command(message_data, message, self.context)
self.wakeup_dispatcher()
dispatch = MessageDispatch(self.context)
return dispatch.start_worker(SCHEDULER_COMMAND, cb)
def test_dispatch_send_recv(ctx):
from eventlet import with_timeout
from eventlet.event import Event
from melkman.messaging import MessageDispatch, always_ack
w = MessageDispatch(ctx)
message_type = "test_dispatch_send_recv"
work_result = Event()
@always_ack
def handler(job, message):
work_result.send(sum(job["values"]))
worker = w.start_worker(message_type, handler)
w.send({"values": [1, 2]}, message_type)
try:
assert with_timeout(2, work_result.wait) == 3
finally:
worker.kill()
worker.wait()
