def async_task(func, *args, **kwargs): """Queue a task for the cluster.""" keywords = kwargs.copy() opt_keys = ( "hook", "group", "save", "sync", "cached", "ack_failure", "iter_count", "iter_cached", "chain", "broker", "timeout", ) q_options = keywords.pop("q_options", {}) # get an id tag = uuid() # build the task package task = { "id": tag[1], "name": keywords.pop("task_name", None) or q_options.pop("task_name", None) or tag[0], "func": func, "args": args, } # push optionals for key in opt_keys: if q_options and key in q_options: task[key] = q_options[key] elif key in keywords: task[key] = keywords.pop(key) # don't serialize the broker broker = task.pop("broker", get_broker()) # overrides if "cached" not in task and Conf.CACHED: task["cached"] = Conf.CACHED if "sync" not in task and Conf.SYNC: task["sync"] = Conf.SYNC if "ack_failure" not in task and Conf.ACK_FAILURES: task["ack_failure"] = Conf.ACK_FAILURES # finalize task["kwargs"] = keywords task["started"] = timezone.now() # signal it pre_enqueue.send(sender="django_q", task=task) # sign it pack = SignedPackage.dumps(task) if task.get("sync", False): return _sync(pack) # push it enqueue_id = broker.enqueue(pack) logger.info(f"Enqueued {enqueue_id}") logger.debug(f"Pushed {tag}") return task["id"]
def async_iter(func, args_iter, **kwargs): """ enqueues a function with iterable arguments """ iter_count = len(args_iter) iter_group = uuid()[1] # clean up the kwargs options = kwargs.get("q_options", kwargs) options.pop("hook", None) options["broker"] = options.get("broker", get_broker()) options["group"] = iter_group options["iter_count"] = iter_count if options.get("cached", None): options["iter_cached"] = options["cached"] options["cached"] = True # save the original arguments broker = options["broker"] broker.cache.set( f"{broker.list_key}:{iter_group}:args", SignedPackage.dumps(args_iter) ) for args in args_iter: if not isinstance(args, tuple): args = (args,) async_task(func, *args, **options) return iter_group
def save_cached(task, broker): task_key = f'{broker.list_key}:{task["id"]}' timeout = task["cached"] if timeout is True: timeout = None try: group = task.get("group", None) iter_count = task.get("iter_count", 0) # if it's a group append to the group list if group: group_key = f"{broker.list_key}:{group}:keys" group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = f"{broker.list_key}:{group}:args" # collate the results into a Task result results = [ SignedPackage.loads(broker.cache.get(k))["result"] for k in group_list ] results.append(task["result"]) task["result"] = results task["id"] = group task["args"] = SignedPackage.loads( broker.cache.get(group_args)) task.pop("iter_count", None) task.pop("group", None) if task.get("iter_cached", None): task["cached"] = task.pop("iter_cached", None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async_task next in a chain if task.get("chain", None): django_q.tasks.async_chain( task["chain"], group=group, cached=task["cached"], sync=task["sync"], broker=broker, ) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def save_cached(task, broker): task_key = '{}:{}'.format(broker.list_key, task['id']) timeout = task['cached'] if timeout is True: timeout = None try: group = task.get('group', None) iter_count = task.get('iter_count', 0) # if it's a group append to the group list if group: group_key = '{}:{}:keys'.format(broker.list_key, group) group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = '{}:{}:args'.format(broker.list_key, group) # collate the results into a Task result results = [ SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list ] results.append(task['result']) task['result'] = results task['id'] = group task['args'] = SignedPackage.loads( broker.cache.get(group_args)) task.pop('iter_count', None) task.pop('group', None) if task.get('iter_cached', None): task['cached'] = task.pop('iter_cached', None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async_task next in a chain if task.get('chain', None): tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def async_task(func, *args, **kwargs): """Queue a task for the cluster.""" keywords = kwargs.copy() opt_keys = ('hook', 'group', 'save', 'sync', 'cached', 'ack_failure', 'iter_count', 'iter_cached', 'chain', 'broker') q_options = keywords.pop('q_options', {}) # get an id tag = uuid() # build the task package task = { 'id': tag[1], 'name': keywords.pop('task_name', None) or q_options.pop('task_name', None) or tag[0], 'func': func, 'args': args } # push optionals for key in opt_keys: if q_options and key in q_options: task[key] = q_options[key] elif key in keywords: task[key] = keywords.pop(key) # don't serialize the broker broker = task.pop('broker', get_broker()) # overrides if 'cached' not in task and Conf.CACHED: task['cached'] = Conf.CACHED if 'sync' not in task and Conf.SYNC: task['sync'] = Conf.SYNC if 'ack_failure' not in task and Conf.ACK_FAILURES: task['ack_failure'] = Conf.ACK_FAILURES # finalize task['kwargs'] = keywords task['started'] = timezone.now() # signal it pre_enqueue.send(sender="django_q", task=task) # sign it pack = SignedPackage.dumps(task) if task.get('sync', False): return _sync(pack) # push it enqueue_id = broker.enqueue(pack) logger.info('Enqueued {}'.format(enqueue_id)) logger.debug('Pushed {}'.format(tag)) return task['id']
def save_cached(task, broker): task_key = '{}:{}'.format(broker.list_key, task['id']) timeout = task['cached'] if timeout is True: timeout = None try: group = task.get('group', None) iter_count = task.get('iter_count', 0) # if it's a group append to the group list if group: group_key = '{}:{}:keys'.format(broker.list_key, group) group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = '{}:{}:args'.format(broker.list_key, group) # collate the results into a Task result results = [SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list] results.append(task['result']) task['result'] = results task['id'] = group task['args'] = SignedPackage.loads(broker.cache.get(group_args)) task.pop('iter_count', None) task.pop('group', None) if task.get('iter_cached', None): task['cached'] = task.pop('iter_cached', None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async next in a chain if task.get('chain', None): tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def async_task(func, *args, **kwargs): """Queue a task for the cluster.""" keywords = kwargs.copy() opt_keys = ( 'hook', 'group', 'save', 'sync', 'cached', 'ack_failure', 'iter_count', 'iter_cached', 'chain', 'broker') q_options = keywords.pop('q_options', {}) # get an id tag = uuid() # build the task package task = {'id': tag[1], 'name': keywords.pop('task_name', None) or q_options.pop('task_name', None) or tag[0], 'func': func, 'args': args} # push optionals for key in opt_keys: if q_options and key in q_options: task[key] = q_options[key] elif key in keywords: task[key] = keywords.pop(key) # don't serialize the broker broker = task.pop('broker', get_broker()) # overrides if 'cached' not in task and Conf.CACHED: task['cached'] = Conf.CACHED if 'sync' not in task and Conf.SYNC: task['sync'] = Conf.SYNC if 'ack_failure' not in task and Conf.ACK_FAILURES: task['ack_failure'] = Conf.ACK_FAILURES # finalize task['kwargs'] = keywords task['started'] = timezone.now() # signal it pre_enqueue.send(sender="django_q", task=task) # sign it pack = SignedPackage.dumps(task) if task.get('sync', False): return _sync(pack) # push it enqueue_id = broker.enqueue(pack) logger.info('Enqueued {}'.format(enqueue_id)) logger.debug('Pushed {}'.format(tag)) return task['id']
def async_iter(func, args_iter, **kwargs): """ enqueues a function with iterable arguments """ iter_count = len(args_iter) iter_group = uuid()[1] # clean up the kwargs options = kwargs.get('q_options', kwargs) options.pop('hook', None) options['broker'] = options.get('broker', get_broker()) options['group'] = iter_group options['iter_count'] = iter_count if options.get('cached', None): options['iter_cached'] = options['cached'] options['cached'] = True # save the original arguments broker = options['broker'] broker.cache.set('{}:{}:args'.format(broker.list_key, iter_group), SignedPackage.dumps(args_iter)) for args in args_iter: if not isinstance(args, tuple): args = (args,) async_task(func, *args, **options) return iter_group
def test_admin_views(admin_client, monkeypatch): monkeypatch.setattr(Conf, 'ORM', 'default') s = schedule('schedule.test') tag = uuid() f = Task.objects.create( id=tag[1], name=tag[0], func='test.fail', started=timezone.now(), stopped=timezone.now(), success=False) tag = uuid() t = Task.objects.create( id=tag[1], name=tag[0], func='test.success', started=timezone.now(), stopped=timezone.now(), success=True) q = OrmQ.objects.create( key='test', payload=SignedPackage.dumps({'id': 1, 'func': 'test', 'name': 'test'})) admin_urls = ( # schedule reverse('admin:django_q_schedule_changelist'), reverse('admin:django_q_schedule_add'), reverse('admin:django_q_schedule_change', args=(s.id,)), reverse('admin:django_q_schedule_history', args=(s.id,)), reverse('admin:django_q_schedule_delete', args=(s.id,)), # success reverse('admin:django_q_success_changelist'), reverse('admin:django_q_success_change', args=(t.id,)), reverse('admin:django_q_success_history', args=(t.id,)), reverse('admin:django_q_success_delete', args=(t.id,)), # failure reverse('admin:django_q_failure_changelist'), reverse('admin:django_q_failure_change', args=(f.id,)), reverse('admin:django_q_failure_history', args=(f.id,)), reverse('admin:django_q_failure_delete', args=(f.id,)), # orm queue reverse('admin:django_q_ormq_changelist'), reverse('admin:django_q_ormq_change', args=(q.id,)), reverse('admin:django_q_ormq_history', args=(q.id,)), reverse('admin:django_q_ormq_delete', args=(q.id,)), ) for url in admin_urls: response = admin_client.get(url) assert response.status_code == 200 # resubmit the failure url = reverse('admin:django_q_failure_changelist') data = {'action': 'retry_failed', '_selected_action': [f.pk]} response = admin_client.post(url, data) assert response.status_code == 302 assert Failure.objects.filter(name=f.id).exists() is False # change q url = reverse('admin:django_q_ormq_change', args=(q.id,)) data = {'key': 'default', 'payload': 'test', 'lock_0': '2015-09-17', 'lock_1': '14:31:51', '_save': 'Save'} response = admin_client.post(url, data) assert response.status_code == 302 # delete q url = reverse('admin:django_q_ormq_delete', args=(q.id,)) data = {'post': 'yes'} response = admin_client.post(url, data) assert response.status_code == 302
def save(self): try: self.broker.set_stat(self.key, SignedPackage.dumps(self, True), 3) except Exception as e: logger.error(e)
def test_admin_views(admin_client, monkeypatch): monkeypatch.setattr(Conf, "ORM", "default") s = schedule("schedule.test") tag = uuid() f = Task.objects.create( id=tag[1], name=tag[0], func="test.fail", started=timezone.now(), stopped=timezone.now(), success=False, ) tag = uuid() t = Task.objects.create( id=tag[1], name=tag[0], func="test.success", started=timezone.now(), stopped=timezone.now(), success=True, ) q = OrmQ.objects.create( key="test", payload=SignedPackage.dumps({ "id": 1, "func": "test", "name": "test" }), ) admin_urls = ( # schedule reverse("admin:django_q_schedule_changelist"), reverse("admin:django_q_schedule_add"), reverse("admin:django_q_schedule_change", args=(s.id, )), reverse("admin:django_q_schedule_history", args=(s.id, )), reverse("admin:django_q_schedule_delete", args=(s.id, )), # success reverse("admin:django_q_success_changelist"), reverse("admin:django_q_success_change", args=(t.id, )), reverse("admin:django_q_success_history", args=(t.id, )), reverse("admin:django_q_success_delete", args=(t.id, )), # failure reverse("admin:django_q_failure_changelist"), reverse("admin:django_q_failure_change", args=(f.id, )), reverse("admin:django_q_failure_history", args=(f.id, )), reverse("admin:django_q_failure_delete", args=(f.id, )), # orm queue reverse("admin:django_q_ormq_changelist"), reverse("admin:django_q_ormq_change", args=(q.id, )), reverse("admin:django_q_ormq_history", args=(q.id, )), reverse("admin:django_q_ormq_delete", args=(q.id, )), ) for url in admin_urls: response = admin_client.get(url) assert response.status_code == 200 # resubmit the failure url = reverse("admin:django_q_failure_changelist") data = {"action": "retry_failed", "_selected_action": [f.pk]} response = admin_client.post(url, data) assert response.status_code == 302 assert Failure.objects.filter(name=f.id).exists() is False # change q url = reverse("admin:django_q_ormq_change", args=(q.id, )) data = { "key": "default", "payload": "test", "lock_0": "2015-09-17", "lock_1": "14:31:51", "_save": "Save", } response = admin_client.post(url, data) assert response.status_code == 302 # delete q url = reverse("admin:django_q_ormq_delete", args=(q.id, )) data = {"post": "yes"} response = admin_client.post(url, data) assert response.status_code == 302
def async_task(func, *pos_args, args=None, kwargs=None, name=None, hook=None, group=None, timeout=None, **q_options): """ Queue a task for the cluster. :param func: Callable function object or string representation of module.function :param pos_args: Positional arguments to provide to func :param args: Positional arguments to provide to func :param kwargs: Keyword arguments to provide to func :param name: Optional custom name of task :param hook: Function to call after task complete (provided Task instance as argument) :param str group: Group identifier (to correlate related tasks) """ func = validate_function(func) hook = validate_function(hook) args = tuple(pos_args or args or tuple()) keywords = kwargs.copy() opt_keys = ( "hook", "group", "save", "sync", # Whether to run the task synchronously "cached", # Remove "ack_failure", # Causes failed tasks to still mark status as complete "iter_count", # Remove "iter_cached", # Remove "chain", # Use prerequisite instead of chain "broker", # dont need "timeout", ) q_options = keywords.pop("q_options", {}) # get an id tag = uuid() # Create task instance task = Task.objects.create( id=tag[1], name=name or tag[0], func=func, args=args, kwargs=kwargs, hook=hook, group=group, ) # push optionals for key in opt_keys: if q_options and key in q_options: task[key] = q_options[key] elif key in keywords: task[key] = keywords.pop(key) # don't serialize the broker broker = task.pop("broker", get_broker()) # overrides if "cached" not in task and Conf.CACHED: task["cached"] = Conf.CACHED if "sync" not in task and Conf.SYNC: task["sync"] = Conf.SYNC if "ack_failure" not in task and Conf.ACK_FAILURES: task["ack_failure"] = Conf.ACK_FAILURES # finalize task["kwargs"] = keywords task["started"] = timezone.now() # signal it pre_enqueue.send(sender="django_q", task=task) # sign it pack = SignedPackage.dumps(task) if task.get("sync", False): return _sync(pack) # push it enqueue_id = broker.enqueue(pack) logger.info(f"Enqueued {enqueue_id}") logger.debug(f"Pushed {tag}") return task["id"]