def async_task(func, *args, **kwargs):
"""Queue a task for the cluster."""
keywords = kwargs.copy()
opt_keys = (
"hook",
"group",
"save",
"sync",
"cached",
"ack_failure",
"iter_count",
"iter_cached",
"chain",
"broker",
"timeout",
)
q_options = keywords.pop("q_options", {})
# get an id
tag = uuid()
# build the task package
task = {
"id":
tag[1],
"name":
keywords.pop("task_name", None) or q_options.pop("task_name", None)
or tag[0],
"func":
func,
"args":
args,
}
# push optionals
for key in opt_keys:
if q_options and key in q_options:
task[key] = q_options[key]
elif key in keywords:
task[key] = keywords.pop(key)
# don't serialize the broker
broker = task.pop("broker", get_broker())
# overrides
if "cached" not in task and Conf.CACHED:
task["cached"] = Conf.CACHED
if "sync" not in task and Conf.SYNC:
task["sync"] = Conf.SYNC
if "ack_failure" not in task and Conf.ACK_FAILURES:
task["ack_failure"] = Conf.ACK_FAILURES
# finalize
task["kwargs"] = keywords
task["started"] = timezone.now()
# signal it
pre_enqueue.send(sender="django_q", task=task)
# sign it
pack = SignedPackage.dumps(task)
if task.get("sync", False):
return _sync(pack)
# push it
enqueue_id = broker.enqueue(pack)
logger.info(f"Enqueued {enqueue_id}")
logger.debug(f"Pushed {tag}")
return task["id"]
def async_iter(func, args_iter, **kwargs):
"""
enqueues a function with iterable arguments
"""
iter_count = len(args_iter)
iter_group = uuid()[1]
# clean up the kwargs
options = kwargs.get("q_options", kwargs)
options.pop("hook", None)
options["broker"] = options.get("broker", get_broker())
options["group"] = iter_group
options["iter_count"] = iter_count
if options.get("cached", None):
options["iter_cached"] = options["cached"]
options["cached"] = True
# save the original arguments
broker = options["broker"]
broker.cache.set(
f"{broker.list_key}:{iter_group}:args", SignedPackage.dumps(args_iter)
)
for args in args_iter:
if not isinstance(args, tuple):
args = (args,)
async_task(func, *args, **options)
return iter_group
def save_cached(task, broker):
task_key = f'{broker.list_key}:{task["id"]}'
timeout = task["cached"]
if timeout is True:
timeout = None
try:
group = task.get("group", None)
iter_count = task.get("iter_count", 0)
# if it's a group append to the group list
if group:
group_key = f"{broker.list_key}:{group}:keys"
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = f"{broker.list_key}:{group}:args"
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))["result"]
for k in group_list
]
results.append(task["result"])
task["result"] = results
task["id"] = group
task["args"] = SignedPackage.loads(
broker.cache.get(group_args))
task.pop("iter_count", None)
task.pop("group", None)
if task.get("iter_cached", None):
task["cached"] = task.pop("iter_cached", None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get("chain", None):
django_q.tasks.async_chain(
task["chain"],
group=group,
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception as e:
logger.error(e)
def save_cached(task, broker):
task_key = '{}:{}'.format(broker.list_key, task['id'])
timeout = task['cached']
if timeout is True:
timeout = None
try:
group = task.get('group', None)
iter_count = task.get('iter_count', 0)
# if it's a group append to the group list
if group:
group_key = '{}:{}:keys'.format(broker.list_key, group)
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = '{}:{}:args'.format(broker.list_key, group)
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))['result']
for k in group_list
]
results.append(task['result'])
task['result'] = results
task['id'] = group
task['args'] = SignedPackage.loads(
broker.cache.get(group_args))
task.pop('iter_count', None)
task.pop('group', None)
if task.get('iter_cached', None):
task['cached'] = task.pop('iter_cached', None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get('chain', None):
tasks.async_chain(task['chain'],
group=group,
cached=task['cached'],
sync=task['sync'],
broker=broker)
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception as e:
logger.error(e)
def async_task(func, *args, **kwargs):
"""Queue a task for the cluster."""
keywords = kwargs.copy()
opt_keys = ('hook', 'group', 'save', 'sync', 'cached', 'ack_failure',
'iter_count', 'iter_cached', 'chain', 'broker')
q_options = keywords.pop('q_options', {})
# get an id
tag = uuid()
# build the task package
task = {
'id':
tag[1],
'name':
keywords.pop('task_name', None) or q_options.pop('task_name', None)
or tag[0],
'func':
func,
'args':
args
}
# push optionals
for key in opt_keys:
if q_options and key in q_options:
task[key] = q_options[key]
elif key in keywords:
task[key] = keywords.pop(key)
# don't serialize the broker
broker = task.pop('broker', get_broker())
# overrides
if 'cached' not in task and Conf.CACHED:
task['cached'] = Conf.CACHED
if 'sync' not in task and Conf.SYNC:
task['sync'] = Conf.SYNC
if 'ack_failure' not in task and Conf.ACK_FAILURES:
task['ack_failure'] = Conf.ACK_FAILURES
# finalize
task['kwargs'] = keywords
task['started'] = timezone.now()
# signal it
pre_enqueue.send(sender="django_q", task=task)
# sign it
pack = SignedPackage.dumps(task)
if task.get('sync', False):
return _sync(pack)
# push it
enqueue_id = broker.enqueue(pack)
logger.info('Enqueued {}'.format(enqueue_id))
logger.debug('Pushed {}'.format(tag))
return task['id']
def save_cached(task, broker):
task_key = '{}:{}'.format(broker.list_key, task['id'])
timeout = task['cached']
if timeout is True:
timeout = None
try:
group = task.get('group', None)
iter_count = task.get('iter_count', 0)
# if it's a group append to the group list
if group:
group_key = '{}:{}:keys'.format(broker.list_key, group)
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = '{}:{}:args'.format(broker.list_key, group)
# collate the results into a Task result
results = [SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list]
results.append(task['result'])
task['result'] = results
task['id'] = group
task['args'] = SignedPackage.loads(broker.cache.get(group_args))
task.pop('iter_count', None)
task.pop('group', None)
if task.get('iter_cached', None):
task['cached'] = task.pop('iter_cached', None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async next in a chain
if task.get('chain', None):
tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker)
# save the task
broker.cache.set(task_key,
SignedPackage.dumps(task),
timeout)
except Exception as e:
logger.error(e)
def async_task(func, *args, **kwargs):
"""Queue a task for the cluster."""
keywords = kwargs.copy()
opt_keys = (
'hook', 'group', 'save', 'sync', 'cached', 'ack_failure', 'iter_count', 'iter_cached', 'chain', 'broker')
q_options = keywords.pop('q_options', {})
# get an id
tag = uuid()
# build the task package
task = {'id': tag[1],
'name': keywords.pop('task_name', None) or q_options.pop('task_name', None) or tag[0],
'func': func,
'args': args}
# push optionals
for key in opt_keys:
if q_options and key in q_options:
task[key] = q_options[key]
elif key in keywords:
task[key] = keywords.pop(key)
# don't serialize the broker
broker = task.pop('broker', get_broker())
# overrides
if 'cached' not in task and Conf.CACHED:
task['cached'] = Conf.CACHED
if 'sync' not in task and Conf.SYNC:
task['sync'] = Conf.SYNC
if 'ack_failure' not in task and Conf.ACK_FAILURES:
task['ack_failure'] = Conf.ACK_FAILURES
# finalize
task['kwargs'] = keywords
task['started'] = timezone.now()
# signal it
pre_enqueue.send(sender="django_q", task=task)
# sign it
pack = SignedPackage.dumps(task)
if task.get('sync', False):
return _sync(pack)
# push it
enqueue_id = broker.enqueue(pack)
logger.info('Enqueued {}'.format(enqueue_id))
logger.debug('Pushed {}'.format(tag))
return task['id']
def async_iter(func, args_iter, **kwargs):
"""
enqueues a function with iterable arguments
"""
iter_count = len(args_iter)
iter_group = uuid()[1]
# clean up the kwargs
options = kwargs.get('q_options', kwargs)
options.pop('hook', None)
options['broker'] = options.get('broker', get_broker())
options['group'] = iter_group
options['iter_count'] = iter_count
if options.get('cached', None):
options['iter_cached'] = options['cached']
options['cached'] = True
# save the original arguments
broker = options['broker']
broker.cache.set('{}:{}:args'.format(broker.list_key, iter_group), SignedPackage.dumps(args_iter))
for args in args_iter:
if not isinstance(args, tuple):
args = (args,)
async_task(func, *args, **options)
return iter_group
def async_iter(func, args_iter, **kwargs):
"""
enqueues a function with iterable arguments
"""
iter_count = len(args_iter)
iter_group = uuid()[1]
# clean up the kwargs
options = kwargs.get('q_options', kwargs)
options.pop('hook', None)
options['broker'] = options.get('broker', get_broker())
options['group'] = iter_group
options['iter_count'] = iter_count
if options.get('cached', None):
options['iter_cached'] = options['cached']
options['cached'] = True
# save the original arguments
broker = options['broker']
broker.cache.set('{}:{}:args'.format(broker.list_key, iter_group), SignedPackage.dumps(args_iter))
for args in args_iter:
if not isinstance(args, tuple):
args = (args,)
async_task(func, *args, **options)
return iter_group
def test_admin_views(admin_client, monkeypatch):
monkeypatch.setattr(Conf, 'ORM', 'default')
s = schedule('schedule.test')
tag = uuid()
f = Task.objects.create(
id=tag[1],
name=tag[0],
func='test.fail',
started=timezone.now(),
stopped=timezone.now(),
success=False)
tag = uuid()
t = Task.objects.create(
id=tag[1],
name=tag[0],
func='test.success',
started=timezone.now(),
stopped=timezone.now(),
success=True)
q = OrmQ.objects.create(
key='test',
payload=SignedPackage.dumps({'id': 1, 'func': 'test', 'name': 'test'}))
admin_urls = (
# schedule
reverse('admin:django_q_schedule_changelist'),
reverse('admin:django_q_schedule_add'),
reverse('admin:django_q_schedule_change', args=(s.id,)),
reverse('admin:django_q_schedule_history', args=(s.id,)),
reverse('admin:django_q_schedule_delete', args=(s.id,)),
# success
reverse('admin:django_q_success_changelist'),
reverse('admin:django_q_success_change', args=(t.id,)),
reverse('admin:django_q_success_history', args=(t.id,)),
reverse('admin:django_q_success_delete', args=(t.id,)),
# failure
reverse('admin:django_q_failure_changelist'),
reverse('admin:django_q_failure_change', args=(f.id,)),
reverse('admin:django_q_failure_history', args=(f.id,)),
reverse('admin:django_q_failure_delete', args=(f.id,)),
# orm queue
reverse('admin:django_q_ormq_changelist'),
reverse('admin:django_q_ormq_change', args=(q.id,)),
reverse('admin:django_q_ormq_history', args=(q.id,)),
reverse('admin:django_q_ormq_delete', args=(q.id,)),
)
for url in admin_urls:
response = admin_client.get(url)
assert response.status_code == 200
# resubmit the failure
url = reverse('admin:django_q_failure_changelist')
data = {'action': 'retry_failed',
'_selected_action': [f.pk]}
response = admin_client.post(url, data)
assert response.status_code == 302
assert Failure.objects.filter(name=f.id).exists() is False
# change q
url = reverse('admin:django_q_ormq_change', args=(q.id,))
data = {'key': 'default', 'payload': 'test', 'lock_0': '2015-09-17', 'lock_1': '14:31:51', '_save': 'Save'}
response = admin_client.post(url, data)
assert response.status_code == 302
# delete q
url = reverse('admin:django_q_ormq_delete', args=(q.id,))
data = {'post': 'yes'}
response = admin_client.post(url, data)
assert response.status_code == 302
def save(self):
try:
self.broker.set_stat(self.key, SignedPackage.dumps(self, True), 3)
except Exception as e:
logger.error(e)
def save(self):
try:
self.broker.set_stat(self.key, SignedPackage.dumps(self, True), 3)
except Exception as e:
logger.error(e)
def test_admin_views(admin_client, monkeypatch):
monkeypatch.setattr(Conf, "ORM", "default")
s = schedule("schedule.test")
tag = uuid()
f = Task.objects.create(
id=tag[1],
name=tag[0],
func="test.fail",
started=timezone.now(),
stopped=timezone.now(),
success=False,
)
tag = uuid()
t = Task.objects.create(
id=tag[1],
name=tag[0],
func="test.success",
started=timezone.now(),
stopped=timezone.now(),
success=True,
)
q = OrmQ.objects.create(
key="test",
payload=SignedPackage.dumps({
"id": 1,
"func": "test",
"name": "test"
}),
)
admin_urls = (
# schedule
reverse("admin:django_q_schedule_changelist"),
reverse("admin:django_q_schedule_add"),
reverse("admin:django_q_schedule_change", args=(s.id, )),
reverse("admin:django_q_schedule_history", args=(s.id, )),
reverse("admin:django_q_schedule_delete", args=(s.id, )),
# success
reverse("admin:django_q_success_changelist"),
reverse("admin:django_q_success_change", args=(t.id, )),
reverse("admin:django_q_success_history", args=(t.id, )),
reverse("admin:django_q_success_delete", args=(t.id, )),
# failure
reverse("admin:django_q_failure_changelist"),
reverse("admin:django_q_failure_change", args=(f.id, )),
reverse("admin:django_q_failure_history", args=(f.id, )),
reverse("admin:django_q_failure_delete", args=(f.id, )),
# orm queue
reverse("admin:django_q_ormq_changelist"),
reverse("admin:django_q_ormq_change", args=(q.id, )),
reverse("admin:django_q_ormq_history", args=(q.id, )),
reverse("admin:django_q_ormq_delete", args=(q.id, )),
)
for url in admin_urls:
response = admin_client.get(url)
assert response.status_code == 200
# resubmit the failure
url = reverse("admin:django_q_failure_changelist")
data = {"action": "retry_failed", "_selected_action": [f.pk]}
response = admin_client.post(url, data)
assert response.status_code == 302
assert Failure.objects.filter(name=f.id).exists() is False
# change q
url = reverse("admin:django_q_ormq_change", args=(q.id, ))
data = {
"key": "default",
"payload": "test",
"lock_0": "2015-09-17",
"lock_1": "14:31:51",
"_save": "Save",
}
response = admin_client.post(url, data)
assert response.status_code == 302
# delete q
url = reverse("admin:django_q_ormq_delete", args=(q.id, ))
data = {"post": "yes"}
response = admin_client.post(url, data)
assert response.status_code == 302
def async_task(func,
*pos_args,
args=None,
kwargs=None,
name=None,
hook=None,
group=None,
timeout=None,
**q_options):
"""
Queue a task for the cluster.
:param func: Callable function object or string representation of module.function
:param pos_args: Positional arguments to provide to func
:param args: Positional arguments to provide to func
:param kwargs: Keyword arguments to provide to func
:param name: Optional custom name of task
:param hook: Function to call after task complete (provided Task instance as argument)
:param str group: Group identifier (to correlate related tasks)
"""
func = validate_function(func)
hook = validate_function(hook)
args = tuple(pos_args or args or tuple())
keywords = kwargs.copy()
opt_keys = (
"hook",
"group",
"save",
"sync", # Whether to run the task synchronously
"cached", # Remove
"ack_failure", # Causes failed tasks to still mark status as complete
"iter_count", # Remove
"iter_cached", # Remove
"chain", # Use prerequisite instead of chain
"broker", # dont need
"timeout",
)
q_options = keywords.pop("q_options", {})
# get an id
tag = uuid()
# Create task instance
task = Task.objects.create(
id=tag[1],
name=name or tag[0],
func=func,
args=args,
kwargs=kwargs,
hook=hook,
group=group,
)
# push optionals
for key in opt_keys:
if q_options and key in q_options:
task[key] = q_options[key]
elif key in keywords:
task[key] = keywords.pop(key)
# don't serialize the broker
broker = task.pop("broker", get_broker())
# overrides
if "cached" not in task and Conf.CACHED:
task["cached"] = Conf.CACHED
if "sync" not in task and Conf.SYNC:
task["sync"] = Conf.SYNC
if "ack_failure" not in task and Conf.ACK_FAILURES:
task["ack_failure"] = Conf.ACK_FAILURES
# finalize
task["kwargs"] = keywords
task["started"] = timezone.now()
# signal it
pre_enqueue.send(sender="django_q", task=task)
# sign it
pack = SignedPackage.dumps(task)
if task.get("sync", False):
return _sync(pack)
# push it
enqueue_id = broker.enqueue(pack)
logger.info(f"Enqueued {enqueue_id}")
logger.debug(f"Pushed {tag}")
return task["id"]