def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None):
"""
Return a list of Tasks for a task group in the cache backend
"""
if not broker:
broker = get_broker()
start = time.time()
if count:
while True:
if count_group_cached(group_id) == count or wait and (time.time() - start) * 1000 >= wait >= 0:
break
time.sleep(0.01)
while True:
group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id))
if group_list:
task_list = []
for task_key in group_list:
task = signing.SignedPackage.loads(broker.cache.get(task_key))
if task['success'] or failures:
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
group=task.get('group'),
success=task['success'])
task_list.append(t)
return task_list
if (time.time() - start) * 1000 >= wait >= 0:
break
time.sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None):
"""
Return the processed task from the cache backend
"""
if not broker:
broker = get_broker()
start = time.time()
while True:
r = broker.cache.get('{}:{}'.format(broker.list_key, task_id))
if r:
task = signing.SignedPackage.loads(r)
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
success=task['success'])
return t
if (time.time() - start) * 1000 >= wait >= 0:
break
time.sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None):
"""
Return the processed task from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get(f"{broker.list_key}:{task_id}")
if r:
task = SignedPackage.loads(r)
return Task(
id=task["id"],
name=task["name"],
func=task["func"],
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
started=task["started"],
stopped=task["stopped"],
result=task["result"],
success=task["success"],
)
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def fetch_group_cached(group_id,
failures=True,
wait=0,
count=None,
broker=None):
"""
Return a list of Tasks for a task group in the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if (count_group_cached(group_id) == count
or wait and (time() - start) * 1000 >= wait >= 0):
break
sleep(0.01)
while True:
group_list = broker.cache.get(f"{broker.list_key}:{group_id}:keys")
if group_list:
task_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task["success"] or failures:
t = Task(
id=task["id"],
name=task["name"],
func=task["func"],
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
started=task["started"],
stopped=task["stopped"],
result=task["result"],
group=task.get("group"),
success=task["success"],
)
task_list.append(t)
return task_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def test_async(broker, admin_user):
broker.list_key = 'cluster_test:q'
broker.delete_queue()
a = async ('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
hook='django_q.tests.test_cluster.assert_result',
broker=broker)
b = async ('django_q.tests.tasks.count_letters2',
WordClass(),
hook='django_q.tests.test_cluster.assert_result',
broker=broker)
# unknown argument
c = async ('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
'oneargumentoomany',
hook='django_q.tests.test_cluster.assert_bad_result',
broker=broker)
# unknown function
d = async ('django_q.tests.tasks.does_not_exist',
WordClass(),
hook='django_q.tests.test_cluster.assert_bad_result',
broker=broker)
# function without result
e = async ('django_q.tests.tasks.countdown', 100000, broker=broker)
# function as instance
f = async (multiply, 753, 2, hook=assert_result, broker=broker)
# model as argument
g = async ('django_q.tests.tasks.get_task_name',
Task(name='John'),
broker=broker)
# args,kwargs, group and broken hook
h = async ('django_q.tests.tasks.word_multiply',
2,
word='django',
hook='fail.me',
broker=broker)
# args unpickle test
j = async ('django_q.tests.tasks.get_user_id',
admin_user,
broker=broker,
group='test_j')
# q_options and save opt_out test
k = async ('django_q.tests.tasks.get_user_id',
admin_user,
q_options={
'broker': broker,
'group': 'test_k',
'save': False,
'timeout': 90
})
# check if everything has a task id
assert isinstance(a, str)
assert isinstance(b, str)
assert isinstance(c, str)
assert isinstance(d, str)
assert isinstance(e, str)
assert isinstance(f, str)
assert isinstance(g, str)
assert isinstance(h, str)
assert isinstance(j, str)
assert isinstance(k, str)
# run the cluster to execute the tasks
task_count = 10
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push the tasks
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
# test wait timeout
assert result(j, wait=10) is None
assert fetch(j, wait=10) is None
assert result_group('test_j', wait=10) is None
assert result_group('test_j', count=2, wait=10) is None
assert fetch_group('test_j', wait=10) is None
assert fetch_group('test_j', count=2, wait=10) is None
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
# Check the results
# task a
result_a = fetch(a)
assert result_a is not None
assert result_a.success is True
assert result(a) == 1506
# task b
result_b = fetch(b)
assert result_b is not None
assert result_b.success is True
assert result(b) == 1506
# task c
result_c = fetch(c)
assert result_c is not None
assert result_c.success is False
# task d
result_d = fetch(d)
assert result_d is not None
assert result_d.success is False
# task e
result_e = fetch(e)
assert result_e is not None
assert result_e.success is True
assert result(e) is None
# task f
result_f = fetch(f)
assert result_f is not None
assert result_f.success is True
assert result(f) == 1506
# task g
result_g = fetch(g)
assert result_g is not None
assert result_g.success is True
assert result(g) == 'John'
# task h
result_h = fetch(h)
assert result_h is not None
assert result_h.success is True
assert result(h) == 12
# task j
result_j = fetch(j)
assert result_j is not None
assert result_j.success is True
assert result_j.result == result_j.args[0].id
# check fetch, result by name
assert fetch(result_j.name) == result_j
assert result(result_j.name) == result_j.result
# groups
assert result_group('test_j')[0] == result_j.result
assert result_j.group_result()[0] == result_j.result
assert result_group('test_j', failures=True)[0] == result_j.result
assert result_j.group_result(failures=True)[0] == result_j.result
assert fetch_group('test_j')[0].id == [result_j][0].id
assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id
assert count_group('test_j') == 1
assert result_j.group_count() == 1
assert count_group('test_j', failures=True) == 0
assert result_j.group_count(failures=True) == 0
assert delete_group('test_j') == 1
assert result_j.group_delete() == 0
deleted_group = delete_group('test_j', tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
deleted_group = result_j.group_delete(tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
# task k should not have been saved
assert fetch(k) is None
assert fetch(k, 100) is None
assert result(k, 100) is None
broker.delete_queue()
def test_enqueue(broker, admin_user):
broker.list_key = "cluster_test:q"
broker.delete_queue()
a = async_task(
"django_q.tests.tasks.count_letters",
DEFAULT_WORDLIST,
hook="django_q.tests.test_cluster.assert_result",
broker=broker,
)
b = async_task(
"django_q.tests.tasks.count_letters2",
WordClass(),
hook="django_q.tests.test_cluster.assert_result",
broker=broker,
)
# unknown argument
c = async_task(
"django_q.tests.tasks.count_letters",
DEFAULT_WORDLIST,
"oneargumentoomany",
hook="django_q.tests.test_cluster.assert_bad_result",
broker=broker,
)
# unknown function
d = async_task(
"django_q.tests.tasks.does_not_exist",
WordClass(),
hook="django_q.tests.test_cluster.assert_bad_result",
broker=broker,
)
# function without result
e = async_task("django_q.tests.tasks.countdown", 100000, broker=broker)
# function as instance
f = async_task(multiply, 753, 2, hook=assert_result, broker=broker)
# model as argument
g = async_task("django_q.tests.tasks.get_task_name",
Task(name="John"),
broker=broker)
# args,kwargs, group and broken hook
h = async_task(
"django_q.tests.tasks.word_multiply",
2,
word="django",
hook="fail.me",
broker=broker,
)
# args unpickle test
j = async_task("django_q.tests.tasks.get_user_id",
admin_user,
broker=broker,
group="test_j")
# q_options and save opt_out test
k = async_task(
"django_q.tests.tasks.get_user_id",
admin_user,
q_options={
"broker": broker,
"group": "test_k",
"save": False,
"timeout": 90
},
)
# test unicode
assert Task(name="Amalia").__str__() == "Amalia"
# check if everything has a task id
assert isinstance(a, str)
assert isinstance(b, str)
assert isinstance(c, str)
assert isinstance(d, str)
assert isinstance(e, str)
assert isinstance(f, str)
assert isinstance(g, str)
assert isinstance(h, str)
assert isinstance(j, str)
assert isinstance(k, str)
# run the cluster to execute the tasks
task_count = 10
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push the tasks
for _ in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put("STOP")
# test wait timeout
assert result(j, wait=10) is None
assert fetch(j, wait=10) is None
assert result_group("test_j", wait=10) is None
assert result_group("test_j", count=2, wait=10) is None
assert fetch_group("test_j", wait=10) is None
assert fetch_group("test_j", count=2, wait=10) is None
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value("f", -1))
assert result_queue.qsize() == task_count
result_queue.put("STOP")
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
# Check the results
# task a
result_a = fetch(a)
assert result_a is not None
assert result_a.success is True
assert result(a) == 1506
# task b
result_b = fetch(b)
assert result_b is not None
assert result_b.success is True
assert result(b) == 1506
# task c
result_c = fetch(c)
assert result_c is not None
assert result_c.success is False
# task d
result_d = fetch(d)
assert result_d is not None
assert result_d.success is False
# task e
result_e = fetch(e)
assert result_e is not None
assert result_e.success is True
assert result(e) is None
# task f
result_f = fetch(f)
assert result_f is not None
assert result_f.success is True
assert result(f) == 1506
# task g
result_g = fetch(g)
assert result_g is not None
assert result_g.success is True
assert result(g) == "John"
# task h
result_h = fetch(h)
assert result_h is not None
assert result_h.success is True
assert result(h) == 12
# task j
result_j = fetch(j)
assert result_j is not None
assert result_j.success is True
assert result_j.result == result_j.args[0].id
# check fetch, result by name
assert fetch(result_j.name) == result_j
assert result(result_j.name) == result_j.result
# groups
assert result_group("test_j")[0] == result_j.result
assert result_j.group_result()[0] == result_j.result
assert result_group("test_j", failures=True)[0] == result_j.result
assert result_j.group_result(failures=True)[0] == result_j.result
assert fetch_group("test_j")[0].id == [result_j][0].id
assert fetch_group("test_j", failures=False)[0].id == [result_j][0].id
assert count_group("test_j") == 1
assert result_j.group_count() == 1
assert count_group("test_j", failures=True) == 0
assert result_j.group_count(failures=True) == 0
assert delete_group("test_j") == 1
assert result_j.group_delete() == 0
deleted_group = delete_group("test_j", tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
deleted_group = result_j.group_delete(tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
# task k should not have been saved
assert fetch(k) is None
assert fetch(k, 100) is None
assert result(k, 100) is None
broker.delete_queue()
def test_async(r):
list_key = 'cluster_test:q'
r.delete(list_key)
a = async ('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
hook='django_q.tests.test_cluster.assert_result',
list_key=list_key)
b = async ('django_q.tests.tasks.count_letters2',
WordClass(),
hook='django_q.tests.test_cluster.assert_result',
list_key=list_key)
# unknown argument
c = async ('django_q.tests.tasks.count_letters',
DEFAULT_WORDLIST,
'oneargumentoomany',
hook='django_q.tests.test_cluster.assert_bad_result',
list_key=list_key)
# unknown function
d = async ('django_q.tests.tasks.does_not_exist',
WordClass(),
hook='django_q.tests.test_cluster.assert_bad_result',
list_key=list_key)
# function without result
e = async ('django_q.tests.tasks.countdown', 100000, list_key=list_key)
# function as instance
f = async (multiply, 753, 2, hook=assert_result, list_key=list_key)
# model as argument
g = async ('django_q.tests.tasks.get_task_name',
Task(name='John'),
list_key=list_key)
# args and kwargs and broken hook
h = async ('django_q.tests.tasks.word_multiply',
2,
word='django',
hook='fail.me',
list_key=list_key,
redis=r)
# check if everything has a task name
assert isinstance(a, str)
assert isinstance(b, str)
assert isinstance(c, str)
assert isinstance(d, str)
assert isinstance(e, str)
assert isinstance(f, str)
assert isinstance(g, str)
assert isinstance(h, str)
# run the cluster to execute the tasks
task_count = 8
assert r.llen(list_key) == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push the tasks
for i in range(task_count):
pusher(task_queue, stop_event, list_key=list_key, r=r)
assert r.llen(list_key) == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('b', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
# Check the results
# task a
result_a = fetch(a)
assert result_a is not None
assert result_a.success is True
assert result(a) == 1506
# task b
result_b = fetch(b)
assert result_b is not None
assert result_b.success is True
assert result(b) == 1506
# task c
result_c = fetch(c)
assert result_c is not None
assert result_c.success is False
# task d
result_d = fetch(d)
assert result_d is not None
assert result_d.success is False
# task e
result_e = fetch(e)
assert result_e is not None
assert result_e.success is True
assert result(e) is None
# task f
result_f = fetch(f)
assert result_f is not None
assert result_f.success is True
assert result(f) == 1506
# task g
result_g = fetch(g)
assert result_g is not None
assert result_g.success is True
assert result(g) == 'John'
# task h
result_h = fetch(h)
assert result_h is not None
assert result_h.success is True
assert result(h) == 12
r.delete(list_key)
