def set_cpu_affinity(n, process_ids, actual=not Conf.TESTING): """ Sets the cpu affinity for the supplied processes. Requires the optional psutil module. :param int n: affinity :param list process_ids: a list of pids :param bool actual: Test workaround for Travis not supporting cpu affinity """ # check if we have the psutil module if not psutil: logger.warning('Skipping cpu affinity because psutil was not found.') return # check if the platform supports cpu_affinity if actual and not hasattr(psutil.Process(process_ids[0]), 'cpu_affinity'): logger.warning('Faking cpu affinity because it is not supported on this platform') actual = False # get the available processors cpu_list = list(range(psutil.cpu_count())) # affinities of 0 or gte cpu_count, equals to no affinity if not n or n >= len(cpu_list): return # spread the workers over the available processors. index = 0 for pid in process_ids: affinity = [] for k in range(n): if index == len(cpu_list): index = 0 affinity.append(cpu_list[index]) index += 1 if psutil.pid_exists(pid): p = psutil.Process(pid) if actual: p.cpu_affinity(affinity) logger.info(_('{} will use cpu {}').format(pid, affinity))
def test_mongo(monkeypatch): monkeypatch.setattr(Conf, 'MONGO', {'host': '127.0.0.1', 'port': 27017}) # check broker broker = get_broker(list_key='mongo_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') tasks = [] for i in range(5): tasks.append(broker.dequeue()[0]) assert broker.lock_size() == 5 for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() broker.delete_queue() assert broker.queue_size() == 0 # check close and autoreconnect broker.close() assert broker.ping() is True
def test_cached(broker): broker.purge_queue() broker.cache.clear() group = 'cache_test' # queue the tests task_id = async ('math.copysign', 1, -1, cached=True, broker=broker) async ('math.copysign', 1, -1, cached=True, broker=broker, group=group) async ('math.copysign', 1, -1, cached=True, broker=broker, group=group) async ('math.copysign', 1, -1, cached=True, broker=broker, group=group) async ('math.copysign', 1, -1, cached=True, broker=broker, group=group) async ('math.copysign', 1, -1, cached=True, broker=broker, group=group) async ('math.popysign', 1, -1, cached=True, broker=broker, group=group) iter_id = async_iter('math.floor', [i for i in range(10)], cached=True) # test wait on cache # test wait timeout assert result(task_id, wait=10, cached=True) is None assert fetch(task_id, wait=10, cached=True) is None assert result_group(group, wait=10, cached=True) is None assert result_group(group, count=2, wait=10, cached=True) is None assert fetch_group(group, wait=10, cached=True) is None assert fetch_group(group, count=2, wait=10, cached=True) is None # run a single inline cluster task_count = 17 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # assert results assert result(task_id, wait=500, cached=True) == -1 assert fetch(task_id, wait=500, cached=True).result == -1 # make sure it's not in the db backend assert fetch(task_id) is None # assert group assert count_group(group, cached=True) == 6 assert count_group(group, cached=True, failures=True) == 1 assert result_group(group, cached=True) == [-1, -1, -1, -1, -1] assert len(result_group(group, cached=True, failures=True)) == 6 assert len(fetch_group(group, cached=True)) == 6 assert len(fetch_group(group, cached=True, failures=False)) == 5 delete_group(group, cached=True) assert count_group(group, cached=True) is None delete_cached(task_id) assert result(task_id, cached=True) is None assert fetch(task_id, cached=True) is None # iter cached assert result(iter_id) is None assert result(iter_id, cached=True) is not None broker.cache.clear()
def test_cached(broker): broker.purge_queue() broker.cache.clear() group = 'cache_test' # queue the tests task_id = async('math.copysign', 1, -1, cached=True, broker=broker) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.copysign', 1, -1, cached=True, broker=broker, group=group) async('math.popysign', 1, -1, cached=True, broker=broker, group=group) iter_id = async_iter('math.floor', [i for i in range(10)], cached=True) # test wait on cache # test wait timeout assert result(task_id, wait=10, cached=True) is None assert fetch(task_id, wait=10, cached=True) is None assert result_group(group, wait=10, cached=True) is None assert result_group(group, count=2, wait=10, cached=True) is None assert fetch_group(group, wait=10, cached=True) is None assert fetch_group(group, count=2, wait=10, cached=True) is None # run a single inline cluster task_count = 17 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') monitor(result_queue) assert result_queue.qsize() == 0 # assert results assert result(task_id, wait=500, cached=True) == -1 assert fetch(task_id, wait=500, cached=True).result == -1 # make sure it's not in the db backend assert fetch(task_id) is None # assert group assert count_group(group, cached=True) == 6 assert count_group(group, cached=True, failures=True) == 1 assert result_group(group, cached=True) == [-1, -1, -1, -1, -1] assert len(result_group(group, cached=True, failures=True)) == 6 assert len(fetch_group(group, cached=True)) == 6 assert len(fetch_group(group, cached=True, failures=False)) == 5 delete_group(group, cached=True) assert count_group(group, cached=True) is None delete_cached(task_id) assert result(task_id, cached=True) is None assert fetch(task_id, cached=True) is None # iter cached assert result(iter_id) is None assert result(iter_id, cached=True) is not None broker.cache.clear()
def test_mongo(monkeypatch): monkeypatch.setattr(Conf, 'MONGO', {'host': '127.0.0.1', 'port': 27017}) # check broker broker = get_broker(list_key='mongo_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') tasks = [] for i in range(5): tasks.append(broker.dequeue()[0]) assert broker.lock_size() == 5 for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() broker.delete_queue() assert broker.queue_size() == 0
def test_info(): info() do_sync() info() for _ in range(24): do_sync() info()
def test_disque(monkeypatch): monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7711']) # check broker broker = get_broker(list_key='disque_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) monkeypatch.setattr(Conf, 'DISQUE_FASTACK', True) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0 # check close and autoreconnect broker.close() assert broker.ping() is True # connection test monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7798', '127.0.0.1:7799']) with pytest.raises(redis.exceptions.ConnectionError): broker.get_connection()
def test_iter(broker): broker.purge_queue() broker.cache.clear() it = [i for i in range(10)] it2 = [(1, -1), (2, -1), (3, -4), (5, 6)] it3 = (1, 2, 3, 4, 5) t = async_iter('math.floor', it, sync=True) t2 = async_iter('math.copysign', it2, sync=True) t3 = async_iter('math.floor', it3, sync=True) t4 = async_iter('math.floor', (1,), sync=True) result_t = result(t) assert result_t is not None task_t = fetch(t) assert task_t.result == result_t assert result(t2) is not None assert result(t3) is not None assert result(t4)[0] == 1 # test iter class i = Iter('math.copysign', sync=True, cached=True) i.append(1, -1) i.append(2, -1) i.append(3, -4) i.append(5, 6) assert i.started is False assert i.length() == 4 assert i.run() is not None assert len(i.result()) == 4 assert len(i.fetch().result) == 4 i.append(1, -7) assert i.result() is None i.run() assert len(i.result()) == 5
def test_iter(broker): broker.purge_queue() broker.cache.clear() it = [i for i in range(10)] it2 = [(1, -1), (2, -1), (3, -4), (5, 6)] it3 = (1, 2, 3, 4, 5) t = async_iter('math.floor', it, sync=True) t2 = async_iter('math.copysign', it2, sync=True) t3 = async_iter('math.floor', it3, sync=True) t4 = async_iter('math.floor', (1, ), sync=True) result_t = result(t) assert result_t is not None task_t = fetch(t) assert task_t.result == result_t assert result(t2) is not None assert result(t3) is not None assert result(t4)[0] == 1 # test iter class i = Iter('math.copysign', sync=True, cached=True) i.append(1, -1) i.append(2, -1) i.append(3, -4) i.append(5, 6) assert i.started is False assert i.length() == 4 assert i.run() is not None assert len(i.result()) == 4 assert len(i.fetch().result) == 4 i.append(1, -7) assert i.result() is None i.run() assert len(i.result()) == 5
def test_ironmq(monkeypatch): monkeypatch.setattr( Conf, 'IRON_MQ', { 'token': os.getenv('IRON_MQ_TOKEN'), 'project_id': os.getenv('IRON_MQ_PROJECT_ID') }) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None # initialize the queue broker.enqueue('test') # clear before we start broker.purge_queue() assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test # monkeypatch.setattr(Conf, 'RETRY', 1) # broker.enqueue('test') # assert broker.dequeue() is not None # sleep(3) # assert broker.dequeue() is not None # task = broker.dequeue()[0] # assert len(task) > 0 # broker.acknowledge(task[0]) # sleep(3) # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() assert broker.dequeue() is None broker.delete_queue() # check close and autoreconnect broker.close() assert broker.ping() is True
def canceled_sqs(monkeypatch): monkeypatch.setattr(Conf, 'SQS', {'aws_region': os.getenv('AWS_REGION'), 'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'), 'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')}) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') sleep(2) # Sometimes SQS is not linear task = broker.dequeue() if not task: pytest.skip('SQS being weird') task = task[0] assert len(task) > 0 broker.acknowledge(task[0]) sleep(2) # delete job monkeypatch.setattr(Conf, 'RETRY', 60) broker.enqueue('test') sleep(1) task = broker.dequeue() if not task: pytest.skip('SQS being weird') task_id = task[0][0] broker.delete(task_id) assert broker.dequeue() is None # fail broker.enqueue('test') while task is None: task = broker.dequeue()[0] broker.fail(task[0]) # bulk test for i in range(10): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 12) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) assert broker.lock_size() == 0 # delete queue broker.enqueue('test') broker.purge_queue() broker.delete_queue()
def test_orm(monkeypatch): monkeypatch.setattr(Conf, 'ORM', 'default') # check broker broker = get_broker(list_key='orm_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) tasks = broker.dequeue() assert broker.lock_size() == Conf.BULK for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0 # check close and autoreconnect broker.close() assert broker.ping() is True
def test_disque(monkeypatch): monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7711']) # check broker broker = get_broker(list_key='disque_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) monkeypatch.setattr(Conf, 'DISQUE_FASTACK', True) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0 # connection test monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7798', '127.0.0.1:7799']) with pytest.raises(redis.exceptions.ConnectionError): broker.get_connection()
def test_orm(monkeypatch): monkeypatch.setattr(Conf, 'ORM', 'default') # check broker broker = get_broker(list_key='orm_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) tasks = broker.dequeue() assert broker.lock_size() == Conf.BULK for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0
def test_ironmq(monkeypatch): monkeypatch.setattr(Conf, 'IRON_MQ', {'token': os.getenv('IRON_MQ_TOKEN'), 'project_id': os.getenv('IRON_MQ_PROJECT_ID')}) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None # initialize the queue broker.enqueue('test') # clear before we start broker.purge_queue() assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test # monkeypatch.setattr(Conf, 'RETRY', 1) # broker.enqueue('test') # assert broker.dequeue() is not None # sleep(3) # assert broker.dequeue() is not None # task = broker.dequeue()[0] # assert len(task) > 0 # broker.acknowledge(task[0]) # sleep(3) # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 5) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() assert broker.dequeue() is None broker.delete_queue()
def spawn_cluster(self): self.pool = [] Stat(self).save() db.connection.close() # spawn worker pool for __ in range(self.pool_size): self.spawn_worker() # spawn auxiliary self.monitor = self.spawn_monitor() self.pusher = self.spawn_pusher() # set worker cpu affinity if needed if psutil and Conf.CPU_AFFINITY: set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def stop(self): Stat(self).save() name = current_process().name logger.info(_('{} stopping cluster processes').format(name)) # Stopping pusher self.event_out.set() # Wait for it to stop while self.pusher.is_alive(): sleep(0.1) Stat(self).save() # Put poison pills in the queue for __ in range(len(self.pool)): self.task_queue.put('STOP') self.task_queue.close() # wait for the task queue to empty self.task_queue.join_thread() # Wait for all the workers to exit while len(self.pool): for p in self.pool: if not p.is_alive(): self.pool.remove(p) sleep(0.1) Stat(self).save() # Finally stop the monitor self.result_queue.put('STOP') self.result_queue.close() # Wait for the result queue to empty self.result_queue.join_thread() logger.info(_('{} waiting for the monitor.').format(name)) # Wait for everything to close or time out count = 0 if not self.timeout: self.timeout = 30 while self.status() == Conf.STOPPING and count < self.timeout * 10: sleep(0.1) Stat(self).save() count += 1 # Final status Stat(self).save()
def test_async(broker, admin_user): broker.list_key = 'cluster_test:q' broker.delete_queue() a = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result', broker=broker) b = async('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result', broker=broker) # unknown argument c = async('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany', hook='django_q.tests.test_cluster.assert_bad_result', broker=broker) # unknown function d = async('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result', broker=broker) # function without result e = async('django_q.tests.tasks.countdown', 100000, broker=broker) # function as instance f = async(multiply, 753, 2, hook=assert_result, broker=broker) # model as argument g = async('django_q.tests.tasks.get_task_name', Task(name='John'), broker=broker) # args,kwargs, group and broken hook h = async('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', broker=broker) # args unpickle test j = async('django_q.tests.tasks.get_user_id', admin_user, broker=broker, group='test_j') # q_options and save opt_out test k = async('django_q.tests.tasks.get_user_id', admin_user, q_options={'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90}) # check if everything has a task id assert isinstance(a, str) assert isinstance(b, str) assert isinstance(c, str) assert isinstance(d, str) assert isinstance(e, str) assert isinstance(f, str) assert isinstance(g, str) assert isinstance(h, str) assert isinstance(j, str) assert isinstance(k, str) # run the cluster to execute the tasks task_count = 10 assert broker.queue_size() == task_count task_queue = Queue() stop_event = Event() stop_event.set() # push the tasks for i in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count task_queue.put('STOP') # test wait timeout assert result(j, wait=10) is None assert fetch(j, wait=10) is None assert result_group('test_j', wait=10) is None assert result_group('test_j', count=2, wait=10) is None assert fetch_group('test_j', wait=10) is None assert fetch_group('test_j', count=2, wait=10) is None # let a worker handle them result_queue = Queue() worker(task_queue, result_queue, Value('f', -1)) assert result_queue.qsize() == task_count result_queue.put('STOP') # store the results monitor(result_queue) assert result_queue.qsize() == 0 # Check the results # task a result_a = fetch(a) assert result_a is not None assert result_a.success is True assert result(a) == 1506 # task b result_b = fetch(b) assert result_b is not None assert result_b.success is True assert result(b) == 1506 # task c result_c = fetch(c) assert result_c is not None assert result_c.success is False # task d result_d = fetch(d) assert result_d is not None assert result_d.success is False # task e result_e = fetch(e) assert result_e is not None assert result_e.success is True assert result(e) is None # task f result_f = fetch(f) assert result_f is not None assert result_f.success is True assert result(f) == 1506 # task g result_g = fetch(g) assert result_g is not None assert result_g.success is True assert result(g) == 'John' # task h result_h = fetch(h) assert result_h is not None assert result_h.success is True assert result(h) == 12 # task j result_j = fetch(j) assert result_j is not None assert result_j.success is True assert result_j.result == result_j.args[0].id # check fetch, result by name assert fetch(result_j.name) == result_j assert result(result_j.name) == result_j.result # groups assert result_group('test_j')[0] == result_j.result assert result_j.group_result()[0] == result_j.result assert result_group('test_j', failures=True)[0] == result_j.result assert result_j.group_result(failures=True)[0] == result_j.result assert fetch_group('test_j')[0].id == [result_j][0].id assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id assert count_group('test_j') == 1 assert result_j.group_count() == 1 assert count_group('test_j', failures=True) == 0 assert result_j.group_count(failures=True) == 0 assert delete_group('test_j') == 1 assert result_j.group_delete() == 0 deleted_group = delete_group('test_j', tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 deleted_group = result_j.group_delete(tasks=True) assert deleted_group is None or deleted_group[0] == 0 # Django 1.9 # task k should not have been saved assert fetch(k) is None assert fetch(k, 100) is None assert result(k, 100) is None broker.delete_queue()
def canceled_sqs(monkeypatch): monkeypatch.setattr( Conf, 'SQS', { 'aws_region': os.getenv('AWS_REGION'), 'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'), 'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY') }) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') sleep(2) # Sometimes SQS is not linear task = broker.dequeue() if not task: pytest.skip('SQS being weird') task = task[0] assert len(task) > 0 broker.acknowledge(task[0]) sleep(2) # delete job monkeypatch.setattr(Conf, 'RETRY', 60) broker.enqueue('test') sleep(1) task = broker.dequeue() if not task: pytest.skip('SQS being weird') task_id = task[0][0] broker.delete(task_id) assert broker.dequeue() is None # fail broker.enqueue('test') while task is None: task = broker.dequeue()[0] broker.fail(task[0]) # bulk test for i in range(10): broker.enqueue('test') monkeypatch.setattr(Conf, 'BULK', 12) tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) assert broker.lock_size() == 0 # delete queue broker.enqueue('test') broker.purge_queue() broker.delete_queue() # check close and autoreconnect broker.close() assert broker.ping() is True