def test_recycle(broker, monkeypatch): # set up the Sentinel broker.list_key = 'test_recycle_test:q' async('django_q.tests.tasks.multiply', 2, 2, broker=broker) async('django_q.tests.tasks.multiply', 2, 2, broker=broker) async('django_q.tests.tasks.multiply', 2, 2, broker=broker) start_event = Event() stop_event = Event() # override settings monkeypatch.setattr(Conf, 'RECYCLE', 2) monkeypatch.setattr(Conf, 'WORKERS', 1) # set a timer to stop the Sentinel threading.Timer(3, stop_event.set).start() s = Sentinel(stop_event, start_event, broker=broker) assert start_event.is_set() assert s.status() == Conf.STOPPED assert s.reincarnations == 1 async('django_q.tests.tasks.multiply', 2, 2, broker=broker) async('django_q.tests.tasks.multiply', 2, 2, broker=broker) task_queue = Queue() result_queue = Queue() # push two tasks pusher(task_queue, stop_event, broker=broker) pusher(task_queue, stop_event, broker=broker) # worker should exit on recycle worker(task_queue, result_queue, Value('f', -1)) # check if the work has been done assert result_queue.qsize() == 2 # save_limit test monkeypatch.setattr(Conf, 'SAVE_LIMIT', 1) result_queue.put('STOP') # run monitor monitor(result_queue) assert Success.objects.count() == Conf.SAVE_LIMIT broker.delete_queue()
def test_max_rss(broker, monkeypatch): # set up the Sentinel broker.list_key = "test_max_rss_test:q" async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker) start_event = Event() stop_event = Event() cluster_id = uuidlib.uuid4() # override settings monkeypatch.setattr(Conf, "MAX_RSS", 40000) monkeypatch.setattr(Conf, "WORKERS", 1) # set a timer to stop the Sentinel threading.Timer(3, stop_event.set).start() s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker) assert start_event.is_set() assert s.status() == Conf.STOPPED assert s.reincarnations == 1 async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker) task_queue = Queue() result_queue = Queue() # push the task pusher(task_queue, stop_event, broker=broker) # worker should exit on recycle worker(task_queue, result_queue, Value("f", -1)) # check if the work has been done assert result_queue.qsize() == 1 # save_limit test monkeypatch.setattr(Conf, "SAVE_LIMIT", 1) result_queue.put("STOP") # run monitor monitor(result_queue) assert Success.objects.count() == Conf.SAVE_LIMIT broker.delete_queue()
def test_cluster(r): list_key = 'cluster_test:q' r.delete(list_key) task = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, list_key=list_key) assert r.llen(list_key) == 1 task_queue = Queue() assert task_queue.qsize() == 0 result_queue = Queue() assert result_queue.qsize() == 0 event = Event() event.set() # Test push pusher(task_queue, event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert r.llen(list_key) == 0 # Test work task_queue.put('STOP') worker(task_queue, result_queue, Value('b', -1)) assert task_queue.qsize() == 0 assert result_queue.qsize() == 1 # Test monitor result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # check result assert result(task) == 1506 r.delete(list_key)
def test_cluster(r): list_key = 'cluster_test:q' r.delete(list_key) task = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, list_key=list_key) assert queue_size(list_key=list_key, r=r) == 1 task_queue = Queue() assert task_queue.qsize() == 0 result_queue = Queue() assert result_queue.qsize() == 0 event = Event() event.set() # Test push pusher(task_queue, event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert queue_size(list_key=list_key, r=r) == 0 # Test work task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) assert task_queue.qsize() == 0 assert result_queue.qsize() == 1 # Test monitor result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # check result assert result(task) == 1506 r.delete(list_key)
def test_recycle(r): # set up the Sentinel list_key = 'test_recycle_test:q' async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) start_event = Event() stop_event = Event() # override settings Conf.RECYCLE = 2 Conf.WORKERS = 1 # set a timer to stop the Sentinel threading.Timer(3, stop_event.set).start() s = Sentinel(stop_event, start_event, list_key=list_key) assert start_event.is_set() assert s.status() == Conf.STOPPED assert s.reincarnations == 1 async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) task_queue = Queue() result_queue = Queue() # push two tasks pusher(task_queue, stop_event, list_key=list_key, r=r) pusher(task_queue, stop_event, list_key=list_key, r=r) # worker should exit on recycle worker(task_queue, result_queue, Value('f', -1)) # check if the work has been done assert result_queue.qsize() == 2 # save_limit test Conf.SAVE_LIMIT = 1 result_queue.put('STOP') # run monitor monitor(result_queue) assert Success.objects.count() == Conf.SAVE_LIMIT r.delete(list_key)
def test_simple_async_report_send(rf, admin_user): broker = get_broker() assert broker.queue_size() == 0 request = rf.get("/") request.query_params = {} request.user = admin_user report = LeaseStatisticReport() response = report.get_response(request) assert response.data assert broker.queue_size() == 1 # Run async task task_queue = Queue() result_queue = Queue() event = Event() event.set() pusher(task_queue, event, broker=broker) assert task_queue.qsize() == 1 assert queue_size(broker=broker) == 0 task_queue.put("STOP") worker(task_queue, result_queue, Value("f", -1)) assert task_queue.qsize() == 0 assert result_queue.qsize() == 1 result_queue.put("STOP") monitor(result_queue) assert result_queue.qsize() == 0 broker.delete_queue() # Test report file have been sent via email assert len(mail.outbox) == 1 assert len(mail.outbox[0].attachments) == 1
def test_recycle(r): # set up the Sentinel list_key = 'test_recycle_test:q' async ('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async ('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async ('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) start_event = Event() stop_event = Event() # override settings Conf.RECYCLE = 2 Conf.WORKERS = 1 # set a timer to stop the Sentinel threading.Timer(3, stop_event.set).start() s = Sentinel(stop_event, start_event, list_key=list_key) assert start_event.is_set() assert s.status() == Conf.STOPPED assert s.reincarnations == 1 async ('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) async ('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r) task_queue = Queue() result_queue = Queue() # push two tasks pusher(task_queue, stop_event, list_key=list_key, r=r) pusher(task_queue, stop_event, list_key=list_key, r=r) # worker should exit on recycle worker(task_queue, result_queue, Value('f', -1)) # check if the work has been done assert result_queue.qsize() == 2 # save_limit test Conf.SAVE_LIMIT = 1 result_queue.put('STOP') # run monitor monitor(result_queue) assert Success.objects.count() == Conf.SAVE_LIMIT r.delete(list_key)
def test_cached(broker): broker.purge_queue() broker.cache.clear() group = 'cache_test' # queue the tests task = async_task('math.copysign', 1, -1, cached=True, broker=broker) task_id = task['id'] async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group) async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group) async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group) async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group) async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group) async_task('math.popysign', 1, -1, cached=True, broker=broker, group=group) iter_id = async_iter('math.floor', [i for i in range(10)], cached=True) # test wait on cache # test wait timeout assert result(task_id, wait=10, cached=True) is None assert fetch(task_id, wait=10, cached=True) is None assert result_group(group, wait=10, cached=True) is None assert result_group(group, count=2, wait=10, cached=True) is None assert fetch_group(group, wait=10, cached=True) is None assert fetch_group(group, count=2, wait=10, cached=True) is None # run a single inline cluster task_count = 17 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # assert results assert result(task_id, wait=500, cached=True) == -1 assert fetch(task_id, wait=500, cached=True).result == -1 # make sure it's not in the db backend assert fetch(task_id) is None # assert group assert count_group(group, cached=True) == 6 assert count_group(group, cached=True, failures=True) == 1 assert result_group(group, cached=True) == [-1, -1, -1, -1, -1] assert len(result_group(group, cached=True, failures=True)) == 6 assert len(fetch_group(group, cached=True)) == 6 assert len(fetch_group(group, cached=True, failures=False)) == 5 delete_group(group, cached=True) assert count_group(group, cached=True) is None delete_cached(task_id) assert result(task_id, cached=True) is None assert fetch(task_id, cached=True) is None # iter cached assert result(iter_id) is None assert result(iter_id, cached=True) is not None broker.cache.clear()
def test_recycle(broker, monkeypatch): # set up the Sentinel broker.list_key = 'test_recycle_test:q' async ('django_q.tests.tasks.multiply', 2, 2, broker=broker) async ('django_q.tests.tasks.multiply', 2, 2, broker=broker) async ('django_q.tests.tasks.multiply', 2, 2, broker=broker) start_event = Event() stop_event = Event() # override settings monkeypatch.setattr(Conf, 'RECYCLE', 2) monkeypatch.setattr(Conf, 'WORKERS', 1) # set a timer to stop the Sentinel threading.Timer(3, stop_event.set).start() s = Sentinel(stop_event, start_event, broker=broker) assert start_event.is_set() assert s.status() == Conf.STOPPED assert s.reincarnations == 1 async ('django_q.tests.tasks.multiply', 2, 2, broker=broker) async ('django_q.tests.tasks.multiply', 2, 2, broker=broker) task_queue = Queue() result_queue = Queue() # push two tasks pusher(task_queue, stop_event, broker=broker) pusher(task_queue, stop_event, broker=broker) # worker should exit on recycle worker(task_queue, result_queue, Value('f', -1)) # check if the work has been done assert result_queue.qsize() == 2 # save_limit test monkeypatch.setattr(Conf, 'SAVE_LIMIT', 1) result_queue.put('STOP') # run monitor monitor(result_queue) assert Success.objects.count() == Conf.SAVE_LIMIT broker.delete_queue()
def test_cluster(broker): broker.list_key = 'cluster_test:q' broker.delete_queue() task = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, broker=broker) assert broker.queue_size() == 1 task_queue = Queue() assert task_queue.qsize() == 0 result_queue = Queue() assert result_queue.qsize() == 0 event = Event() event.set() # Test push pusher(task_queue, event, broker=broker) assert task_queue.qsize() == 1 assert queue_size(broker=broker) == 0 # Test work task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) assert task_queue.qsize() == 0 assert result_queue.qsize() == 1 # Test monitor result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # check result assert result(task) == 1506 broker.delete_queue()
def test_post_execute_signal(self, broker): broker.list_key = "post_execute_test:q" broker.delete_queue() self.signal_was_called: bool = False self.task: Optional[dict] = None self.func = None def handler(sender, task, **kwargs): self.signal_was_called = True self.task = task post_execute.connect(handler) task_id = async_task("math.copysign", 1, -1, broker=broker) task_queue = Queue() result_queue = Queue() event = Event() event.set() pusher(task_queue, event, broker=broker) task_queue.put("STOP") worker(task_queue, result_queue, Value("f", -1)) result_queue.put("STOP") monitor(result_queue, broker) broker.delete_queue() assert self.signal_was_called is True assert self.task.get("id") == task_id assert self.task.get("result") == -1 post_execute.disconnect(handler)
def test_cached(broker): broker.purge_queue() broker.cache.clear() group = 'cache_test' # queue the tests task_id = async('math.copysign', 1, -1, cached=True, broker=broker) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.popysign', 1, -1, cached=True, broker=broker, group=group) iter_id = async_iter('math.floor', [i for i in range(10)], cached=True) # test wait on cache # test wait timeout assert result(task_id, wait=10, cached=True) is None assert fetch(task_id, wait=10, cached=True) is None assert result_group(group, wait=10, cached=True) is None assert result_group(group, count=2, wait=10, cached=True) is None assert fetch_group(group, wait=10, cached=True) is None assert fetch_group(group, count=2, wait=10, cached=True) is None # run a single inline cluster task_count = 17 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # assert results assert result(task_id, wait=500, cached=True) == -1 assert fetch(task_id, wait=500, cached=True).result == -1 # make sure it's not in the db backend assert fetch(task_id) is None # assert group assert count_group(group, cached=True) == 6 assert count_group(group, cached=True, failures=True) == 1 assert result_group(group, cached=True) == [-1, -1, -1, -1, -1] assert len(result_group(group, cached=True, failures=True)) == 6 assert len(fetch_group(group, cached=True)) == 6 assert len(fetch_group(group, cached=True, failures=False)) == 5 delete_group(group, cached=True) assert count_group(group, cached=True) is None delete_cached(task_id) assert result(task_id, cached=True) is None assert fetch(task_id, cached=True) is None # iter cached assert result(iter_id) is None assert result(iter_id, cached=True) is not None broker.cache.clear()
def test_acknowledge_failure_override(): class VerifyAckMockBroker(Broker): def __init__(self, *args, **kwargs): super(VerifyAckMockBroker, self).__init__(*args, **kwargs) self.acknowledgements = {} def acknowledge(self, task_id): count = self.acknowledgements.get(task_id, 0) self.acknowledgements[task_id] = count + 1 tag = uuid() task_fail_ack = { 'id': tag[1], 'name': tag[0], 'ack_id': 'test_fail_ack_id', 'ack_failure': True, 'func': 'math.copysign', 'args': (1, -1), 'kwargs': {}, 'started': timezone.now(), 'stopped': timezone.now(), 'success': False, 'result': None } tag = uuid() task_fail_no_ack = task_fail_ack.copy() task_fail_no_ack.update({ 'id': tag[1], 'name': tag[0], 'ack_id': 'test_fail_no_ack_id' }) del task_fail_no_ack['ack_failure'] tag = uuid() task_success_ack = task_fail_ack.copy() task_success_ack.update({ 'id': tag[1], 'name': tag[0], 'ack_id': 'test_success_ack_id', 'success': True, }) del task_success_ack['ack_failure'] result_queue = Queue() result_queue.put(task_fail_ack) result_queue.put(task_fail_no_ack) result_queue.put(task_success_ack) result_queue.put('STOP') broker = VerifyAckMockBroker(list_key='key') monitor(result_queue, broker) assert broker.acknowledgements.get('test_fail_ack_id') == 1 assert broker.acknowledgements.get('test_fail_no_ack_id') is None assert broker.acknowledgements.get('test_success_ack_id') == 1
def test_acknowledge_failure_override(): class VerifyAckMockBroker(Broker): def __init__(self, *args, **kwargs): super(VerifyAckMockBroker, self).__init__(*args, **kwargs) self.acknowledgements = {} def acknowledge(self, task_id): count = self.acknowledgements.get(task_id, 0) self.acknowledgements[task_id] = count + 1 tag = uuid() task_fail_ack = { "id": tag[1], "name": tag[0], "ack_id": "test_fail_ack_id", "ack_failure": True, "func": "math.copysign", "args": (1, -1), "kwargs": {}, "started": timezone.now(), "stopped": timezone.now(), "success": False, "result": None, } tag = uuid() task_fail_no_ack = task_fail_ack.copy() task_fail_no_ack.update({ "id": tag[1], "name": tag[0], "ack_id": "test_fail_no_ack_id" }) del task_fail_no_ack["ack_failure"] tag = uuid() task_success_ack = task_fail_ack.copy() task_success_ack.update({ "id": tag[1], "name": tag[0], "ack_id": "test_success_ack_id", "success": True, }) del task_success_ack["ack_failure"] result_queue = Queue() result_queue.put(task_fail_ack) result_queue.put(task_fail_no_ack) result_queue.put(task_success_ack) result_queue.put("STOP") broker = VerifyAckMockBroker(list_key="key") monitor(result_queue, broker) assert broker.acknowledgements.get("test_fail_ack_id") == 1 assert broker.acknowledgements.get("test_fail_no_ack_id") is None assert broker.acknowledgements.get("test_success_ack_id") == 1
def _sync(pack): # Python 2.6 is unable to handle this import on top of the file # because it creates a circular dependency between tasks and cluster from django_q.cluster import worker, monitor """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) return task['id']
def _sync(pack): """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task['id']
def _sync(pack): """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task['id']
def test_scheduler(r): list_key = 'scheduler_test:q' r.delete(list_key) schedule = create_schedule('math.copysign', 1, -1, hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # run scheduler scheduler(list_key=list_key) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert r.llen(list_key) == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 for t in Schedule.TYPE: schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result' ) assert schedule is not None assert schedule.last_run() is None scheduler()
def _sync(pack): """Simulate a package travelling through the cluster.""" from django_q.cluster import monitor, worker task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put("STOP") worker(task_queue, result_queue, Value("f", -1)) result_queue.put("STOP") monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task["id"]
def test_scheduler(r): list_key = 'scheduler_test:q' r.delete(list_key) schedule = create_schedule('math.copysign', 1, -1, hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # run scheduler scheduler(list_key=list_key) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert r.llen(list_key) == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule.refresh_from_db() assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 for t in Schedule.TYPE: schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result') assert schedule is not None assert schedule.last_run() is None
def test_scheduler(broker): broker.list_key = 'scheduler_test:q' broker.delete_queue() schedule = create_schedule('math.copysign', 1, -1, name='test math', hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # run scheduler scheduler(broker=broker) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, broker=broker) assert task_queue.qsize() == 1 assert broker.queue_size() == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True assert schedule.next_run < arrow.get(timezone.now()).replace(hours=+1) task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 # Once schedule with delete once_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.ONCE, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(once_schedule, 'pk') is True # negative repeats always_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.DAILY, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(always_schedule, 'pk') is True # Minute schedule minute_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.MINUTES, minutes=10) assert hasattr(minute_schedule, 'pk') is True # All other types for t in Schedule.TYPE: schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result' ) assert schedule is not None assert schedule.last_run() is None scheduler(broker=broker) # via model Schedule.objects.create(func='django_q.tests.tasks.word_multiply', args='2', kwargs='word="django"', schedule_type=Schedule.DAILY ) # scheduler scheduler(broker=broker) # ONCE schedule should be deleted assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False # Catch up On Conf.CATCH_UP = True now = timezone.now() schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='catch_up', schedule_type=Schedule.HOURLY, next_run=timezone.now() - timedelta(hours=12), repeats=-1 ) scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run < now # Catch up off Conf.CATCH_UP = False scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run > now # Done broker.delete_queue()
def test_scheduler(broker, monkeypatch): broker.list_key = "scheduler_test:q" broker.delete_queue() schedule = create_schedule( "math.copysign", 1, -1, name="test math", hook="django_q.tests.tasks.result", schedule_type=Schedule.HOURLY, repeats=1, ) assert schedule.last_run() is None # check duplicate constraint with pytest.raises(IntegrityError): schedule = create_schedule( "math.copysign", 1, -1, name="test math", hook="django_q.tests.tasks.result", schedule_type=Schedule.HOURLY, repeats=1, ) # run scheduler scheduler(broker=broker) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, broker=broker) assert task_queue.qsize() == 1 assert broker.queue_size() == 0 task_queue.put("STOP") # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value("b", -1)) assert result_queue.qsize() == 1 result_queue.put("STOP") # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True assert schedule.next_run < arrow.get(timezone.now()).shift(hours=+1) task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 # Once schedule with delete once_schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=Schedule.ONCE, repeats=-1, hook="django_q.tests.tasks.result", ) assert hasattr(once_schedule, "pk") is True # negative repeats always_schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=Schedule.DAILY, repeats=-1, hook="django_q.tests.tasks.result", ) assert hasattr(always_schedule, "pk") is True # Minute schedule minute_schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=Schedule.MINUTES, minutes=10, ) assert hasattr(minute_schedule, "pk") is True # Cron schedule cron_schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=Schedule.CRON, cron="0 22 * * 1-5", ) assert hasattr(cron_schedule, "pk") is True assert cron_schedule.full_clean() is None assert cron_schedule.__str__() == "django_q.tests.tasks.word_multiply" with pytest.raises(ValidationError): create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=Schedule.CRON, cron="0 22 * * 1-12", ) # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: continue schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="django", schedule_type=t[0], repeats=1, hook="django_q.tests.tasks.result", ) assert schedule is not None assert schedule.last_run() is None scheduler(broker=broker) # via model Schedule.objects.create( func="django_q.tests.tasks.word_multiply", args="2", kwargs='word="django"', schedule_type=Schedule.DAILY, ) # scheduler scheduler(broker=broker) # ONCE schedule should be deleted assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False # Catch up On monkeypatch.setattr(Conf, "CATCH_UP", True) now = timezone.now() schedule = create_schedule( "django_q.tests.tasks.word_multiply", 2, word="catch_up", schedule_type=Schedule.HOURLY, next_run=timezone.now() - timedelta(hours=12), repeats=-1, ) scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run < now # Catch up off monkeypatch.setattr(Conf, "CATCH_UP", False) scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run > now # Done broker.delete_queue() monkeypatch.setattr(Conf, "PREFIX", "some_cluster_name") # create a schedule on another cluster schedule = create_schedule( "math.copysign", 1, -1, name="test schedule on a another cluster", hook="django_q.tests.tasks.result", schedule_type=Schedule.HOURLY, cluster="some_other_cluster_name", repeats=1, ) # run scheduler scheduler(broker=broker) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, broker=broker) # queue must be empty assert task_queue.qsize() == 0 monkeypatch.setattr(Conf, "PREFIX", "default") # create a schedule on the same cluster schedule = create_schedule( "math.copysign", 1, -1, name="test schedule with no cluster", hook="django_q.tests.tasks.result", schedule_type=Schedule.HOURLY, cluster="default", repeats=1, ) # run scheduler scheduler(broker=broker) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, broker=broker) # queue must contain a task assert task_queue.qsize() == 1
def test_async(broker, admin_user): broker.list_key = 'cluster_test:q' broker.delete_queue() a = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result', broker=broker) b = async ('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result', broker=broker) # unknown argument c = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany', hook='django_q.tests.test_cluster.assert_bad_result', broker=broker) # unknown function d = async ('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result', broker=broker) # function without result e = async ('django_q.tests.tasks.countdown', 100000, broker=broker) # function as instance f = async (multiply, 753, 2, hook=assert_result, broker=broker) # model as argument g = async ('django_q.tests.tasks.get_task_name', Task(name='John'), broker=broker) # args,kwargs, group and broken hook h = async ('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', broker=broker) # args unpickle test j = async ('django_q.tests.tasks.get_user_id', admin_user, broker=broker, group='test_j') # q_options and save opt_out test k = async ('django_q.tests.tasks.get_user_id', admin_user, q_options={ 'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90 }) # check if everything has a task id assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) assert isinstance(j, str) assert isinstance(k, str) # run the cluster to execute the tasks task_count = 10 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') # test wait timeout assert result(j, wait=10) is None assert fetch(j, wait=10) is None assert result_group('test_j', wait=10) is None assert result_group('test_j', count=2, wait=10) is None assert fetch_group('test_j', wait=10) is None assert fetch_group('test_j', count=2, wait=10) is None # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == 'John' # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 # task j result_j = fetch(j) assert result_j is not None assert result_j.success is True assert result_j.result == result_j.args[0].id # check fetch, result by name assert fetch(result_j.name) == result_j assert result(result_j.name) == result_j.result # groups assert result_group('test_j')[0] == result_j.result assert result_j.group_result()[0] == result_j.result assert result_group('test_j', failures=True)[0] == result_j.result assert result_j.group_result(failures=True)[0] == result_j.result assert fetch_group('test_j')[0].id == [result_j][0].id assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id assert count_group('test_j') == 1 assert result_j.group_count() == 1 assert count_group('test_j', failures=True) == 0 assert result_j.group_count(failures=True) == 0 assert delete_group('test_j') == 1 assert result_j.group_delete() == 0 deleted_group = delete_group('test_j', tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 deleted_group = result_j.group_delete(tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 # task k should not have been saved assert fetch(k) is None assert fetch(k, 100) is None assert result(k, 100) is None broker.delete_queue()
def test_scheduler(r): list_key = 'scheduler_test:q' r.delete(list_key) schedule = create_schedule('math.copysign', 1, -1, name='test math', hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # run scheduler scheduler(list_key=list_key) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert r.llen(list_key) == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True assert schedule.next_run < arrow.get(timezone.now()).replace(hours=+1) task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 # Once schedule with delete once_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.ONCE, repeats=-1, hook='django_q.tests.tasks.result') assert hasattr(once_schedule, 'pk') is True # negative repeats always_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.DAILY, repeats=-1, hook='django_q.tests.tasks.result') assert hasattr(always_schedule, 'pk') is True # All other types for t in Schedule.TYPE: schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result') assert schedule is not None assert schedule.last_run() is None scheduler(list_key=list_key) scheduler(list_key=list_key) # ONCE schedule should be deleted assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False r.delete(list_key)
def test_scheduler(r): list_key = 'scheduler_test:q' r.delete(list_key) schedule = create_schedule('math.copysign', 1, -1, name='test math', hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # run scheduler scheduler(list_key=list_key) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, list_key=list_key, r=r) assert task_queue.qsize() == 1 assert r.llen(list_key) == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True assert schedule.next_run < arrow.get(timezone.now()).replace(hours=+1) task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 # Once schedule with delete once_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.ONCE, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(once_schedule, 'pk') is True # negative repeats always_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.DAILY, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(always_schedule, 'pk') is True # All other types for t in Schedule.TYPE: schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result' ) assert schedule is not None assert schedule.last_run() is None scheduler(list_key=list_key) scheduler(list_key=list_key) # ONCE schedule should be deleted assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False r.delete(list_key)
def test_async(r): list_key = 'cluster_test:q' r.delete(list_key) a = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result', list_key=list_key) b = async('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result', list_key=list_key) # unknown argument c = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany', hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key) # unknown function d = async('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key) # function without result e = async('django_q.tests.tasks.countdown', 100000, list_key=list_key) # function as instance f = async(multiply, 753, 2, hook=assert_result, list_key=list_key) # model as argument g = async('django_q.tests.tasks.get_task_name', Task(name='John'), list_key=list_key) # args and kwargs and broken hook h = async('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', list_key=list_key, redis=r) # check if everything has a task name assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) # run the cluster to execute the tasks task_count = 8 assert r.llen(list_key) == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for i in range(task_count): pusher(task_queue, stop_event, list_key=list_key, r=r) assert r.llen(list_key) == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == 'John' # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 r.delete(list_key)
def test_scheduler(broker, monkeypatch): broker.list_key = 'scheduler_test:q' broker.delete_queue() schedule = create_schedule('math.copysign', 1, -1, name='test math', hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) assert schedule.last_run() is None # check duplicate constraint with pytest.raises(IntegrityError): schedule = create_schedule('math.copysign', 1, -1, name='test math', hook='django_q.tests.tasks.result', schedule_type=Schedule.HOURLY, repeats=1) # run scheduler scheduler(broker=broker) # set up the workflow task_queue = Queue() stop_event = Event() stop_event.set() # push it pusher(task_queue, stop_event, broker=broker) assert task_queue.qsize() == 1 assert broker.queue_size() == 0 task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == 1 result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.repeats == 0 assert schedule.last_run() is not None assert schedule.success() is True assert schedule.next_run < arrow.get(timezone.now()).shift(hours=+1) task = fetch(schedule.task) assert task is not None assert task.success is True assert task.result < 0 # Once schedule with delete once_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.ONCE, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(once_schedule, 'pk') is True # negative repeats always_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.DAILY, repeats=-1, hook='django_q.tests.tasks.result' ) assert hasattr(always_schedule, 'pk') is True # Minute schedule minute_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.MINUTES, minutes=10) assert hasattr(minute_schedule, 'pk') is True # Cron schedule cron_schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.CRON, cron="0 22 * * 1-5") assert hasattr(cron_schedule, 'pk') is True assert cron_schedule.full_clean() is None assert cron_schedule.__str__() == 'django_q.tests.tasks.word_multiply' with pytest.raises(ValidationError): create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=Schedule.CRON, cron="0 22 * * 1-12") # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: continue schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', schedule_type=t[0], repeats=1, hook='django_q.tests.tasks.result' ) assert schedule is not None assert schedule.last_run() is None scheduler(broker=broker) # via model Schedule.objects.create(func='django_q.tests.tasks.word_multiply', args='2', kwargs='word="django"', schedule_type=Schedule.DAILY ) # scheduler scheduler(broker=broker) # ONCE schedule should be deleted assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False # Catch up On monkeypatch.setattr(Conf, 'CATCH_UP', True) now = timezone.now() schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='catch_up', schedule_type=Schedule.HOURLY, next_run=timezone.now() - timedelta(hours=12), repeats=-1 ) scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run < now # Catch up off monkeypatch.setattr(Conf, 'CATCH_UP', False) scheduler(broker=broker) schedule = Schedule.objects.get(pk=schedule.pk) assert schedule.next_run > now # Done broker.delete_queue()
def test_enqueue(broker, admin_user): broker.list_key = "cluster_test:q" broker.delete_queue() a = async_task( "django_q.tests.tasks.count_letters", DEFAULT_WORDLIST, hook="django_q.tests.test_cluster.assert_result", broker=broker, ) b = async_task( "django_q.tests.tasks.count_letters2", WordClass(), hook="django_q.tests.test_cluster.assert_result", broker=broker, ) # unknown argument c = async_task( "django_q.tests.tasks.count_letters", DEFAULT_WORDLIST, "oneargumentoomany", hook="django_q.tests.test_cluster.assert_bad_result", broker=broker, ) # unknown function d = async_task( "django_q.tests.tasks.does_not_exist", WordClass(), hook="django_q.tests.test_cluster.assert_bad_result", broker=broker, ) # function without result e = async_task("django_q.tests.tasks.countdown", 100000, broker=broker) # function as instance f = async_task(multiply, 753, 2, hook=assert_result, broker=broker) # model as argument g = async_task("django_q.tests.tasks.get_task_name", Task(name="John"), broker=broker) # args,kwargs, group and broken hook h = async_task( "django_q.tests.tasks.word_multiply", 2, word="django", hook="fail.me", broker=broker, ) # args unpickle test j = async_task("django_q.tests.tasks.get_user_id", admin_user, broker=broker, group="test_j") # q_options and save opt_out test k = async_task( "django_q.tests.tasks.get_user_id", admin_user, q_options={ "broker": broker, "group": "test_k", "save": False, "timeout": 90 }, ) # test unicode assert Task(name="Amalia").__str__() == "Amalia" # check if everything has a task id assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) assert isinstance(j, str) assert isinstance(k, str) # run the cluster to execute the tasks task_count = 10 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for _ in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put("STOP") # test wait timeout assert result(j, wait=10) is None assert fetch(j, wait=10) is None assert result_group("test_j", wait=10) is None assert result_group("test_j", count=2, wait=10) is None assert fetch_group("test_j", wait=10) is None assert fetch_group("test_j", count=2, wait=10) is None # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value("f", -1)) assert result_queue.qsize() == task_count result_queue.put("STOP") # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == "John" # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 # task j result_j = fetch(j) assert result_j is not None assert result_j.success is True assert result_j.result == result_j.args[0].id # check fetch, result by name assert fetch(result_j.name) == result_j assert result(result_j.name) == result_j.result # groups assert result_group("test_j")[0] == result_j.result assert result_j.group_result()[0] == result_j.result assert result_group("test_j", failures=True)[0] == result_j.result assert result_j.group_result(failures=True)[0] == result_j.result assert fetch_group("test_j")[0].id == [result_j][0].id assert fetch_group("test_j", failures=False)[0].id == [result_j][0].id assert count_group("test_j") == 1 assert result_j.group_count() == 1 assert count_group("test_j", failures=True) == 0 assert result_j.group_count(failures=True) == 0 assert delete_group("test_j") == 1 assert result_j.group_delete() == 0 deleted_group = delete_group("test_j", tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 deleted_group = result_j.group_delete(tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 # task k should not have been saved assert fetch(k) is None assert fetch(k, 100) is None assert result(k, 100) is None broker.delete_queue()
def test_async(r): list_key = 'cluster_test:q' r.delete(list_key) a = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result', list_key=list_key) b = async ('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result', list_key=list_key) # unknown argument c = async ('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany', hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key) # unknown function d = async ('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key) # function without result e = async ('django_q.tests.tasks.countdown', 100000, list_key=list_key) # function as instance f = async (multiply, 753, 2, hook=assert_result, list_key=list_key) # model as argument g = async ('django_q.tests.tasks.get_task_name', Task(name='John'), list_key=list_key) # args and kwargs and broken hook h = async ('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', list_key=list_key, redis=r) # check if everything has a task name assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) # run the cluster to execute the tasks task_count = 8 assert r.llen(list_key) == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for i in range(task_count): pusher(task_queue, stop_event, list_key=list_key, r=r) assert r.llen(list_key) == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('b', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == 'John' # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 r.delete(list_key)
def test_async(r, admin_user): list_key = 'cluster_test:q' r.delete(list_key) a = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result', list_key=list_key, redis=r) b = async('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result', list_key=list_key, redis=r) # unknown argument c = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany', hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key, redis=r) # unknown function d = async('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result', list_key=list_key, redis=r) # function without result e = async('django_q.tests.tasks.countdown', 100000, list_key=list_key, redis=r) # function as instance f = async(multiply, 753, 2, hook=assert_result, list_key=list_key, redis=r) # model as argument g = async('django_q.tests.tasks.get_task_name', Task(name='John'), list_key=list_key, redis=r) # args,kwargs, group and broken hook h = async('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', list_key=list_key, redis=r) # args unpickle test j = async('django_q.tests.tasks.get_user_id', admin_user, list_key=list_key, group='test_j', redis=r) # q_options and save opt_out test k = async('django_q.tests.tasks.get_user_id', admin_user, q_options={'list_key': list_key, 'group': 'test_k', 'redis': r, 'save': False, 'timeout': 90}) # check if everything has a task id assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) assert isinstance(j, str) assert isinstance(k, str) # run the cluster to execute the tasks task_count = 10 assert queue_size(list_key=list_key, r=r) == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for i in range(task_count): pusher(task_queue, stop_event, list_key=list_key, r=r) assert queue_size(list_key=list_key, r=r) == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == 'John' # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 # task j result_j = fetch(j) assert result_j is not None assert result_j.success is True assert result_j.result == result_j.args[0].id # check fetch, result by name assert fetch(result_j.name) == result_j assert result(result_j.name) == result_j.result # groups assert result_group('test_j') == [result_j.result] assert result_group('test_j', failures=True) == [result_j.result] assert fetch_group('test_j')[0].id == [result_j][0].id assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id assert count_group('test_j') == 1 assert count_group('test_j', failures=True) == 0 assert delete_group('test_j') == 1 assert delete_group('test_j', tasks=True) is None # task k should not have been saved assert fetch(k) is None r.delete(list_key)