def named_queue(url):
"""Named queues facilitate discrete queues on a single backend."""
foo_state = []
def foo_func(arg):
foo_state.append(arg)
foo_broker = get_broker(url, 'foo')
foo_broker.expose(foo_func)
bar_state = []
def bar_func(arg):
bar_state.append(arg)
bar_broker = get_broker(url, 'bar')
bar_broker.expose(bar_func)
with thread_worker(foo_broker), thread_worker(bar_broker):
# -- task-invoking code, usually another process --
f = get_queue(url, 'foo')
f.foo_func(1)
b = get_queue(url, 'bar')
b.bar_func(2)
eventually((lambda:(foo_state, bar_state)), ([1], [2]))
def result_status(url):
"""Deferred results can be queried for task status.
A lock is used to control state interactions between the producer
and the worker for illustration purposes only. This type of
lock-step interaction is not normally needed or even desired.
"""
lock = TimeoutLock(locked=True)
def func(arg):
lock.acquire()
return arg
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
# -- task-invoking code, usually another process --
q = get_queue(url)
res = q.func('arg')
eventually((lambda:res.status), const.ENQUEUED)
eq_(repr(res), "<Deferred func [default:%s] enqueued>" % res.id)
lock.release()
eventually((lambda:res.status), const.PROCESSING)
eq_(repr(res), "<Deferred func [default:%s] processing>" % res.id)
lock.release()
assert res.wait(WAIT), repr(res)
eq_(repr(res), "<Deferred func [default:%s] success>" % res.id)
eq_(res.value, 'arg')
def named_queue(url):
"""Named queues facilitate discrete queues on a single backend."""
foo_state = []
def foo_func(arg):
foo_state.append(arg)
foo_broker = get_broker(url, 'foo')
foo_broker.expose(foo_func)
bar_state = []
def bar_func(arg):
bar_state.append(arg)
bar_broker = get_broker(url, 'bar')
bar_broker.expose(bar_func)
with thread_worker(foo_broker), thread_worker(bar_broker):
# -- task-invoking code, usually another process --
f = get_queue(url, 'foo')
f.foo_func(1)
b = get_queue(url, 'bar')
b.bar_func(2)
eventually((lambda: (foo_state, bar_state)), ([1], [2]))
def expose_method(url):
"""Object methods can be exposed too, not just functions."""
class Database(object):
"""stateful storage"""
value = None
def update(self, value):
self.value = value
class TaskObj(object):
"""object with task definitions"""
def __init__(self, db):
self.db = db
def update_value(self, value):
self.db.update(value)
db = Database()
obj = TaskObj(db)
broker = get_broker(url)
broker.expose(obj.update_value)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.update_value(2)
eventually((lambda: db.value), 2)
def task_namespaces(url):
"""Task namepsaces are used to arrange tasks similar to the Python
package/module hierarchy.
"""
state = set()
__name__ = 'module.path'
ts = TaskSpace(__name__)
@ts.task
def foo():
state.add('foo')
@ts.task
def bar(arg):
state.add(arg)
broker = get_broker(url)
broker.expose(ts)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.module.path.foo()
q.module.path.bar(1)
eventually((lambda: state), {'foo', 1})
def result_status(url):
"""Deferred results can be queried for task status.
A lock is used to control state interactions between the producer
and the worker for illustration purposes only. This type of
lock-step interaction is not normally needed or even desired.
"""
lock = TimeoutLock(locked=True)
def func(arg):
lock.acquire()
return arg
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
# -- task-invoking code, usually another process --
q = get_queue(url)
res = q.func('arg')
eventually((lambda: res.status), const.ENQUEUED)
eq_(repr(res), "<Deferred func [default:%s] enqueued>" % res.id)
lock.release()
eventually((lambda: res.status), const.PROCESSING)
eq_(repr(res), "<Deferred func [default:%s] processing>" % res.id)
lock.release()
assert res.wait(WAIT), repr(res)
eq_(repr(res), "<Deferred func [default:%s] success>" % res.id)
eq_(res.value, 'arg')
def expose_method(url):
"""Object methods can be exposed too, not just functions."""
class Database(object):
"""stateful storage"""
value = None
def update(self, value):
self.value = value
class TaskObj(object):
"""object with task definitions"""
def __init__(self, db):
self.db = db
def update_value(self, value):
self.db.update(value)
db = Database()
obj = TaskObj(db)
broker = get_broker(url)
broker.expose(obj.update_value)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.update_value(2)
eventually((lambda:db.value), 2)
def task_namespaces(url):
"""Task namepsaces are used to arrange tasks similar to the Python
package/module hierarchy.
"""
state = set()
__name__ = 'module.path'
ts = TaskSpace(__name__)
@ts.task
def foo():
state.add('foo')
@ts.task
def bar(arg):
state.add(arg)
broker = get_broker(url)
broker.expose(ts)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.module.path.foo()
q.module.path.bar(1)
eventually((lambda:state), {'foo', 1})
def func(arg, lock=None):
# signal func started
touch(join(tmp, 'func.started'))
# wait for unlock
eventually(reader(tmp, 'func.unlock'), '')
# write data to file
touch(join(tmp, 'func.out'), arg)
log.debug('func complete')
def func(arg, lock=None):
# signal func started
touch(join(tmp, 'func.started'))
# wait for unlock
eventually(reader(tmp, 'func.unlock'), '')
# write data to file
touch(join(tmp, 'func.out'), arg)
log.debug('func complete')
def simple(url):
"""A simple example demonstrating WorQ mechanics"""
state = []
def func(arg):
state.append(arg)
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.func('arg')
eventually((lambda: state), ['arg'])
def simple(url):
"""A simple example demonstrating WorQ mechanics"""
state = []
def func(arg):
state.append(arg)
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.func('arg')
eventually((lambda:state), ['arg'])
def more_namespaces(url):
state = set()
foo = TaskSpace('foo')
bar = TaskSpace('foo.bar')
baz = TaskSpace('foo.bar.baz')
@foo.task
def join(arg):
state.add('foo-join %s' % arg)
@bar.task
def kick(arg):
state.add('bar-kick %s' % arg)
@baz.task
def join(arg):
state.add('baz-join %s' % arg)
@baz.task
def kick(arg):
state.add('baz-kick %s' % arg)
broker = get_broker(url)
broker.expose(foo)
broker.expose(bar)
broker.expose(baz)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.foo.join(1)
q.foo.bar.kick(2)
q.foo.bar.baz.join(3)
q.foo.bar.baz.kick(4)
eventually((lambda:state), {
'foo-join 1',
'bar-kick 2',
'baz-join 3',
'baz-kick 4',
})
def more_namespaces(url):
state = set()
foo = TaskSpace('foo')
bar = TaskSpace('foo.bar')
baz = TaskSpace('foo.bar.baz')
@foo.task
def join(arg):
state.add('foo-join %s' % arg)
@bar.task
def kick(arg):
state.add('bar-kick %s' % arg)
@baz.task
def join(arg):
state.add('baz-join %s' % arg)
@baz.task
def kick(arg):
state.add('baz-kick %s' % arg)
broker = get_broker(url)
broker.expose(foo)
broker.expose(bar)
broker.expose(baz)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
q.foo.join(1)
q.foo.bar.kick(2)
q.foo.bar.baz.join(3)
q.foo.bar.baz.kick(4)
eventually((lambda: state), {
'foo-join 1',
'bar-kick 2',
'baz-join 3',
'baz-kick 4',
})
def test_Queue_len(url):
lock = TimeoutLock(locked=True)
def func(arg=None):
pass
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
q = get_queue(url)
eq_(len(q), 0)
r0 = q.func()
eq_(len(q), 1)
r1 = q.func()
r2 = q.func(r1)
eq_(len(q), 3)
eventually((lambda: lock.locked), True)
lock.release()
assert r0.wait(timeout=WAIT), repr(r0)
eq_(len(q), 2)
eventually((lambda: lock.locked), True)
lock.release()
eventually((lambda: lock.locked), True)
lock.release()
assert r2.wait(timeout=WAIT), repr(r2)
eq_(len(q), 0)
def test_Queue_len(url):
lock = TimeoutLock(locked=True)
def func(arg=None):
pass
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
q = get_queue(url)
eq_(len(q), 0)
r0 = q.func()
eq_(len(q), 1)
r1 = q.func()
r2 = q.func(r1)
eq_(len(q), 3)
eventually((lambda:lock.locked), True)
lock.release()
assert r0.wait(timeout=WAIT), repr(r0)
eq_(len(q), 2)
eventually((lambda:lock.locked), True)
lock.release()
eventually((lambda:lock.locked), True)
lock.release()
assert r2.wait(timeout=WAIT), repr(r2)
eq_(len(q), 0)
def test_WorkerPool_worker_shutdown_on_parent_die(url):
with tempdir() as tmp:
logpath = join(tmp, 'output.log')
proc = run_in_subprocess(worker_pool, url,
WorkerPool_worker_shutdown_on_parent_die_init,
(tmp, logpath))
with printlog(logpath), force_kill_on_exit(proc):
res = get_queue(url).getpid()
assert res.wait(WAIT), repr(res)
os.kill(proc.pid, signal.SIGKILL) # force kill pool master
eventually(proc.is_alive, False, timeout=WAIT)
try:
eventually(pid_running(res.value), False,
timeout=WAIT, poll_interval=0.1)
except Exception:
os.kill(res.value, signal.SIGTERM) # clean up
raise
def ignore_result(url):
"""Tell the queue to ignore the task result when the result is not
important. This is done by creating a ``Task`` object with custom
options for more efficient queue operation.
"""
state = []
def func(arg):
state.append(arg)
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
f = Task(q.func, ignore_result=True)
res = f(3)
eq_(res, None) # verify that we did not get a deferred result
eventually((lambda: state), [3])
def test_WorkerPool_worker_shutdown_on_parent_die(url):
with tempdir() as tmp:
logpath = join(tmp, 'output.log')
proc = run_in_subprocess(worker_pool, url,
WorkerPool_worker_shutdown_on_parent_die_init,
(tmp, logpath))
with printlog(logpath), force_kill_on_exit(proc):
res = get_queue(url).getpid()
assert res.wait(WAIT), repr(res)
os.kill(proc.pid, signal.SIGKILL) # force kill pool master
eventually(proc.is_alive, False, timeout=WAIT)
try:
eventually(pid_running(res.value), False,
timeout=WAIT, poll_interval=0.1)
except Exception:
os.kill(res.value, signal.SIGTERM) # clean up
raise
def ignore_result(url):
"""Tell the queue to ignore the task result when the result is not
important. This is done by creating a ``Task`` object with custom
options for more efficient queue operation.
"""
state = []
def func(arg):
state.append(arg)
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker):
# -- task-invoking code, usually another process --
q = get_queue(url)
f = Task(q.func, ignore_result=True)
res = f(3)
eq_(res, None) # verify that we did not get a deferred result
eventually((lambda:state), [3])
def Broker_duplicate_task_id(url, identifier):
lock = TimeoutLock(locked=True)
state = []
def func(arg):
lock.acquire()
return arg
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
q = get_queue(url)
task = Task(q.func, id=identifier)
res = task(1)
eventually((lambda: res.status), const.ENQUEUED)
msg = 'func [default:int] cannot enqueue task with duplicate id'
with assert_raises(DuplicateTask, msg):
task(2)
lock.release()
eventually((lambda: res.status), const.PROCESSING)
msg = 'func [default:int] cannot enqueue task with duplicate id'
with assert_raises(DuplicateTask, msg):
task(3)
lock.release()
assert res.wait(timeout=WAIT), repr(res)
eq_(res.value, 1)
res = task(4)
eventually((lambda: res.status), const.ENQUEUED)
lock.release()
eventually((lambda: res.status), const.PROCESSING)
lock.release()
assert res.wait(timeout=WAIT), repr(res)
eq_(res.value, 4)
def Broker_duplicate_task_id(url, identifier):
lock = TimeoutLock(locked=True)
state = []
def func(arg):
lock.acquire()
return arg
broker = get_broker(url)
broker.expose(func)
with thread_worker(broker, lock):
q = get_queue(url)
task = Task(q.func, id=identifier)
res = task(1)
eventually((lambda:res.status), const.ENQUEUED)
msg = 'func [default:int] cannot enqueue task with duplicate id'
with assert_raises(DuplicateTask, msg):
task(2)
lock.release()
eventually((lambda:res.status), const.PROCESSING)
msg = 'func [default:int] cannot enqueue task with duplicate id'
with assert_raises(DuplicateTask, msg):
task(3)
lock.release()
assert res.wait(timeout=WAIT), repr(res)
eq_(res.value, 1)
res = task(4)
eventually((lambda:res.status), const.ENQUEUED)
lock.release()
eventually((lambda:res.status), const.PROCESSING)
lock.release()
assert res.wait(timeout=WAIT), repr(res)
eq_(res.value, 4)
def test_WorkerPool_sigterm(url):
with tempdir() as tmp:
logpath = join(tmp, 'output.log')
proc = run_in_subprocess(worker_pool, url,
WorkerPool_sigterm_init, (tmp, logpath), workers=3)
with printlog(logpath), force_kill_on_exit(proc):
q = get_queue(url)
q.func('text')
eventually(reader(tmp, 'func.started'), '')
proc.terminate() # signal pool shutdown
touch(join(tmp, 'func.unlock')) # allow func to proceed
eventually(reader(tmp, 'func.out'), 'text')
eventually(verify_shutdown(proc), True, timeout=WAIT)
def test_WorkerPool_sigterm(url):
with tempdir() as tmp:
logpath = join(tmp, 'output.log')
proc = run_in_subprocess(worker_pool, url,
WorkerPool_sigterm_init, (tmp, logpath), workers=3)
with printlog(logpath), force_kill_on_exit(proc):
q = get_queue(url)
q.func('text')
eventually(reader(tmp, 'func.started'), '')
proc.terminate() # signal pool shutdown
touch(join(tmp, 'func.unlock')) # allow func to proceed
eventually(reader(tmp, 'func.out'), 'text')
eventually(verify_shutdown(proc), True, timeout=WAIT)