def pusher(task_queue: Queue, event: Event, broker: Broker = None):
"""
Pulls tasks of the broker and puts them in the task queue
:type broker:
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
if not broker:
broker = get_broker()
logger.info(_(f"{current_process().name} pushing tasks at {current_process().pid}"))
while True:
try:
task_set = broker.dequeue()
except Exception as e:
logger.error(e, traceback.format_exc())
# broker probably crashed. Let the sentinel handle it.
sleep(10)
break
if task_set:
for task in task_set:
ack_id = task[0]
# unpack the task
try:
task = SignedPackage.loads(task[1])
except (TypeError, BadSignature) as e:
logger.error(e, traceback.format_exc())
broker.fail(ack_id)
continue
task["ack_id"] = ack_id
task_queue.put(task)
logger.debug(_(f"queueing from {broker.list_key}"))
if event.is_set():
break
logger.info(_(f"{current_process().name} stopped pushing tasks"))
def __init__(
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.pid = current_process().pid
self.cluster_id = cluster_id
self.parent_pid = get_ppid()
self.name = current_process().name
self.broker = broker or get_broker()
self.reincarnations = 0
self.tob = timezone.now()
self.stop_event = stop_event
self.start_event = start_event
self.pool_size = Conf.WORKERS
self.pool = []
self.timeout = timeout
self.task_queue = (Queue(
maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue())
self.result_queue = Queue()
self.event_out = Event()
self.monitor = None
self.pusher = None
if start:
self.start()
def _sync(pack):
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = signing.SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
cluster.worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
cluster.monitor(result_queue)
return task['id']
def worker(task_queue: Queue,
result_queue: Queue,
timer: Value,
timeout: int = Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:param timeout: number of seconds wait for a worker to finish.
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(_(f"{name} ready for work at {current_process().pid}"))
task_count = 0
if timeout is None:
timeout = -1
# Start reading the task queue
for task in iter(task_queue.get, "STOP"):
result = None
timer.value = -1 # Idle
task_count += 1
# Get the function from the task
logger.info(_(f'{name} processing [{task["name"]}]'))
f = task["func"]
# if it's not an instance try to get it from the string
if not callable(task["func"]):
f = pydoc.locate(f)
close_old_django_connections()
timer_value = task.pop("timeout", timeout)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task["args"], **task["kwargs"])
result = (res, True)
except Exception as e:
result = (f"{e} : {traceback.format_exc()}", False)
if error_reporter:
error_reporter.report()
if task.get("sync", False):
raise
with timer.get_lock():
# Process result
task["result"] = result[0]
task["success"] = result[1]
task["stopped"] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE or rss_check():
timer.value = -2 # Recycled
break
logger.info(_(f"{name} stopped doing work"))
def test_cached(broker):
broker.purge_queue()
broker.cache.clear()
group = 'cache_test'
# queue the tests
task_id = async('math.copysign', 1, -1, cached=True, broker=broker)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.popysign', 1, -1, cached=True, broker=broker, group=group)
iter_id = async_iter('math.floor', [i for i in range(10)], cached=True)
# test wait on cache
# test wait timeout
assert result(task_id, wait=10, cached=True) is None
assert fetch(task_id, wait=10, cached=True) is None
assert result_group(group, wait=10, cached=True) is None
assert result_group(group, count=2, wait=10, cached=True) is None
assert fetch_group(group, wait=10, cached=True) is None
assert fetch_group(group, count=2, wait=10, cached=True) is None
# run a single inline cluster
task_count = 17
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
monitor(result_queue)
assert result_queue.qsize() == 0
# assert results
assert result(task_id, wait=500, cached=True) == -1
assert fetch(task_id, wait=500, cached=True).result == -1
# make sure it's not in the db backend
assert fetch(task_id) is None
# assert group
assert count_group(group, cached=True) == 6
assert count_group(group, cached=True, failures=True) == 1
assert result_group(group, cached=True) == [-1, -1, -1, -1, -1]
assert len(result_group(group, cached=True, failures=True)) == 6
assert len(fetch_group(group, cached=True)) == 6
assert len(fetch_group(group, cached=True, failures=False)) == 5
delete_group(group, cached=True)
assert count_group(group, cached=True) is None
delete_cached(task_id)
assert result(task_id, cached=True) is None
assert fetch(task_id, cached=True) is None
# iter cached
assert result(iter_id) is None
assert result(iter_id, cached=True) is not None
broker.cache.clear()
def _sync(pack):
# Python 2.6 is unable to handle this import on top of the file
# because it creates a circular dependency between tasks and cluster
from django_q.cluster import worker, monitor
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
monitor(result_queue)
return task['id']
def test_post_execute_signal(self, broker):
broker.list_key = "post_execute_test:q"
broker.delete_queue()
self.signal_was_called: bool = False
self.task: Optional[dict] = None
self.func = None
def handler(sender, task, **kwargs):
self.signal_was_called = True
self.task = task
post_execute.connect(handler)
task_id = async_task("math.copysign", 1, -1, broker=broker)
task_queue = Queue()
result_queue = Queue()
event = Event()
event.set()
pusher(task_queue, event, broker=broker)
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue, broker)
broker.delete_queue()
assert self.signal_was_called is True
assert self.task.get("id") == task_id
assert self.task.get("result") == -1
post_execute.disconnect(handler)
def test_acknowledge_failure_override():
class VerifyAckMockBroker(Broker):
def __init__(self, *args, **kwargs):
super(VerifyAckMockBroker, self).__init__(*args, **kwargs)
self.acknowledgements = {}
def acknowledge(self, task_id):
count = self.acknowledgements.get(task_id, 0)
self.acknowledgements[task_id] = count + 1
tag = uuid()
task_fail_ack = {
"id": tag[1],
"name": tag[0],
"ack_id": "test_fail_ack_id",
"ack_failure": True,
"func": "math.copysign",
"args": (1, -1),
"kwargs": {},
"started": timezone.now(),
"stopped": timezone.now(),
"success": False,
"result": None,
}
tag = uuid()
task_fail_no_ack = task_fail_ack.copy()
task_fail_no_ack.update({
"id": tag[1],
"name": tag[0],
"ack_id": "test_fail_no_ack_id"
})
del task_fail_no_ack["ack_failure"]
tag = uuid()
task_success_ack = task_fail_ack.copy()
task_success_ack.update({
"id": tag[1],
"name": tag[0],
"ack_id": "test_success_ack_id",
"success": True,
})
del task_success_ack["ack_failure"]
result_queue = Queue()
result_queue.put(task_fail_ack)
result_queue.put(task_fail_no_ack)
result_queue.put(task_success_ack)
result_queue.put("STOP")
broker = VerifyAckMockBroker(list_key="key")
monitor(result_queue, broker)
assert broker.acknowledgements.get("test_fail_ack_id") == 1
assert broker.acknowledgements.get("test_fail_no_ack_id") is None
assert broker.acknowledgements.get("test_success_ack_id") == 1
def test_acknowledge_failure_override():
class VerifyAckMockBroker(Broker):
def __init__(self, *args, **kwargs):
super(VerifyAckMockBroker, self).__init__(*args, **kwargs)
self.acknowledgements = {}
def acknowledge(self, task_id):
count = self.acknowledgements.get(task_id, 0)
self.acknowledgements[task_id] = count + 1
tag = uuid()
task_fail_ack = {
'id': tag[1],
'name': tag[0],
'ack_id': 'test_fail_ack_id',
'ack_failure': True,
'func': 'math.copysign',
'args': (1, -1),
'kwargs': {},
'started': timezone.now(),
'stopped': timezone.now(),
'success': False,
'result': None
}
tag = uuid()
task_fail_no_ack = task_fail_ack.copy()
task_fail_no_ack.update({
'id': tag[1],
'name': tag[0],
'ack_id': 'test_fail_no_ack_id'
})
del task_fail_no_ack['ack_failure']
tag = uuid()
task_success_ack = task_fail_ack.copy()
task_success_ack.update({
'id': tag[1],
'name': tag[0],
'ack_id': 'test_success_ack_id',
'success': True,
})
del task_success_ack['ack_failure']
result_queue = Queue()
result_queue.put(task_fail_ack)
result_queue.put(task_fail_no_ack)
result_queue.put(task_success_ack)
result_queue.put('STOP')
broker = VerifyAckMockBroker(list_key='key')
monitor(result_queue, broker)
assert broker.acknowledgements.get('test_fail_ack_id') == 1
assert broker.acknowledgements.get('test_fail_no_ack_id') is None
assert broker.acknowledgements.get('test_success_ack_id') == 1
def test_bad_secret(broker, monkeypatch):
broker.list_key = "test_bad_secret:q"
async_task("math.copysign", 1, -1, broker=broker)
stop_event = Event()
stop_event.set()
start_event = Event()
cluster_id = uuidlib.uuid4()
s = Sentinel(stop_event,
start_event,
cluster_id=cluster_id,
broker=broker,
start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all()
assert len(stat) == 0
assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None
task_queue = Queue()
pusher(task_queue, stop_event, broker=broker)
result_queue = Queue()
task_queue.put("STOP")
worker(
task_queue,
result_queue,
Value("f", -1),
)
assert result_queue.qsize() == 0
broker.delete_queue()
def test_max_rss(broker, monkeypatch):
# set up the Sentinel
broker.list_key = 'test_max_rss_test:q'
async_task('django_q.tests.tasks.multiply', 2, 2, broker=broker)
start_event = Event()
stop_event = Event()
cluster_id = uuidlib.uuid4()
# override settings
monkeypatch.setattr(Conf, 'MAX_RSS', 40000)
monkeypatch.setattr(Conf, 'WORKERS', 1)
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async_task('django_q.tests.tasks.multiply', 2, 2, broker=broker)
task_queue = Queue()
result_queue = Queue()
# push the task
pusher(task_queue, stop_event, broker=broker)
# worker should exit on recycle
worker(task_queue, result_queue, Value('f', -1))
# check if the work has been done
assert result_queue.qsize() == 1
# save_limit test
monkeypatch.setattr(Conf, 'SAVE_LIMIT', 1)
result_queue.put('STOP')
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
broker.delete_queue()
def __init__(self, stop_event, start_event, broker=None, timeout=Conf.TIMEOUT, start=True):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.pid = current_process().pid
self.parent_pid = get_ppid()
self.name = current_process().name
self.broker = broker or get_broker()
self.reincarnations = 0
self.tob = timezone.now()
self.stop_event = stop_event
self.start_event = start_event
self.pool_size = Conf.WORKERS
self.pool = []
self.timeout = timeout
self.task_queue = Queue(maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue()
self.result_queue = Queue()
self.event_out = Event()
self.monitor = None
self.pusher = None
if start:
self.start()
def test_bad_secret(broker, monkeypatch):
broker.list_key = 'test_bad_secret:q'
async_task('math.copysign', 1, -1, broker=broker)
stop_event = Event()
stop_event.set()
start_event = Event()
s = Sentinel(stop_event, start_event, broker=broker, start=False)
Stat(s).save()
# change the SECRET
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
stat = Stat.get_all()
assert len(stat) == 0
assert Stat.get(s.parent_pid) is None
task_queue = Queue()
pusher(task_queue, stop_event, broker=broker)
result_queue = Queue()
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1), )
assert result_queue.qsize() == 0
broker.delete_queue()
def test_recycle(broker, monkeypatch):
# set up the Sentinel
broker.list_key = "test_recycle_test:q"
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
start_event = Event()
stop_event = Event()
cluster_id = uuidlib.uuid4()
# override settings
monkeypatch.setattr(Conf, "RECYCLE", 2)
monkeypatch.setattr(Conf, "WORKERS", 1)
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
task_queue = Queue()
result_queue = Queue()
# push two tasks
pusher(task_queue, stop_event, broker=broker)
pusher(task_queue, stop_event, broker=broker)
# worker should exit on recycle
worker(task_queue, result_queue, Value("f", -1))
# check if the work has been done
assert result_queue.qsize() == 2
# save_limit test
monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
result_queue.put("STOP")
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
broker.delete_queue()
def _sync(pack):
"""Simulate a package travelling through the cluster."""
from django_q.cluster import monitor, worker
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task["id"]
def test_enqueue(broker, admin_user):
broker.list_key = 'cluster_test:q'
broker.delete_queue()
a = async_task('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, hook='django_q.tests.test_cluster.assert_result',
broker=broker)
b = async_task('django_q.tests.tasks.count_letters2', WordClass(), hook='django_q.tests.test_cluster.assert_result',
broker=broker)
# unknown argument
c = async_task('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, 'oneargumentoomany',
hook='django_q.tests.test_cluster.assert_bad_result', broker=broker)
# unknown function
d = async_task('django_q.tests.tasks.does_not_exist', WordClass(), hook='django_q.tests.test_cluster.assert_bad_result',
broker=broker)
# function without result
e = async_task('django_q.tests.tasks.countdown', 100000, broker=broker)
# function as instance
f = async_task(multiply, 753, 2, hook=assert_result, broker=broker)
# model as argument
g = async_task('django_q.tests.tasks.get_task_name', Task(name='John'), broker=broker)
# args,kwargs, group and broken hook
h = async_task('django_q.tests.tasks.word_multiply', 2, word='django', hook='fail.me', broker=broker)
# args unpickle test
j = async_task('django_q.tests.tasks.get_user_id', admin_user, broker=broker, group='test_j')
# q_options and save opt_out test
k = async_task('django_q.tests.tasks.get_user_id', admin_user,
q_options={'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90})
# test unicode
assert Task(name='Amalia').__str__()=='Amalia'
# check if everything has a task id
assert isinstance(a, str)
assert isinstance(b, str)
assert isinstance(c, str)
assert isinstance(d, str)
assert isinstance(e, str)
assert isinstance(f, str)
assert isinstance(g, str)
assert isinstance(h, str)
assert isinstance(j, str)
assert isinstance(k, str)
# run the cluster to execute the tasks
task_count = 10
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push the tasks
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
# test wait timeout
assert result(j, wait=10) is None
assert fetch(j, wait=10) is None
assert result_group('test_j', wait=10) is None
assert result_group('test_j', count=2, wait=10) is None
assert fetch_group('test_j', wait=10) is None
assert fetch_group('test_j', count=2, wait=10) is None
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
# Check the results
# task a
result_a = fetch(a)
assert result_a is not None
assert result_a.success is True
assert result(a) == 1506
# task b
result_b = fetch(b)
assert result_b is not None
assert result_b.success is True
assert result(b) == 1506
# task c
result_c = fetch(c)
assert result_c is not None
assert result_c.success is False
# task d
result_d = fetch(d)
assert result_d is not None
assert result_d.success is False
# task e
result_e = fetch(e)
assert result_e is not None
assert result_e.success is True
assert result(e) is None
# task f
result_f = fetch(f)
assert result_f is not None
assert result_f.success is True
assert result(f) == 1506
# task g
result_g = fetch(g)
assert result_g is not None
assert result_g.success is True
assert result(g) == 'John'
# task h
result_h = fetch(h)
assert result_h is not None
assert result_h.success is True
assert result(h) == 12
# task j
result_j = fetch(j)
assert result_j is not None
assert result_j.success is True
assert result_j.result == result_j.args[0].id
# check fetch, result by name
assert fetch(result_j.name) == result_j
assert result(result_j.name) == result_j.result
# groups
assert result_group('test_j')[0] == result_j.result
assert result_j.group_result()[0] == result_j.result
assert result_group('test_j', failures=True)[0] == result_j.result
assert result_j.group_result(failures=True)[0] == result_j.result
assert fetch_group('test_j')[0].id == [result_j][0].id
assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id
assert count_group('test_j') == 1
assert result_j.group_count() == 1
assert count_group('test_j', failures=True) == 0
assert result_j.group_count(failures=True) == 0
assert delete_group('test_j') == 1
assert result_j.group_delete() == 0
deleted_group = delete_group('test_j', tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
deleted_group = result_j.group_delete(tasks=True)
assert deleted_group is None or deleted_group[0] == 0 # Django 1.9
# task k should not have been saved
assert fetch(k) is None
assert fetch(k, 100) is None
assert result(k, 100) is None
broker.delete_queue()
def test_scheduler(broker, monkeypatch):
broker.list_key = "scheduler_test:q"
broker.delete_queue()
schedule = create_schedule(
"math.copysign",
1,
-1,
name="test math",
hook="django_q.tests.tasks.result",
schedule_type=Schedule.HOURLY,
repeats=1,
)
assert schedule.last_run() is None
# check duplicate constraint
with pytest.raises(IntegrityError):
schedule = create_schedule(
"math.copysign",
1,
-1,
name="test math",
hook="django_q.tests.tasks.result",
schedule_type=Schedule.HOURLY,
repeats=1,
)
# run scheduler
scheduler(broker=broker)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, broker=broker)
assert task_queue.qsize() == 1
assert broker.queue_size() == 0
task_queue.put("STOP")
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value("b", -1))
assert result_queue.qsize() == 1
result_queue.put("STOP")
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.repeats == 0
assert schedule.last_run() is not None
assert schedule.success() is True
assert schedule.next_run < arrow.get(timezone.now()).shift(hours=+1)
task = fetch(schedule.task)
assert task is not None
assert task.success is True
assert task.result < 0
# Once schedule with delete
once_schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=Schedule.ONCE,
repeats=-1,
hook="django_q.tests.tasks.result",
)
assert hasattr(once_schedule, "pk") is True
# negative repeats
always_schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=Schedule.DAILY,
repeats=-1,
hook="django_q.tests.tasks.result",
)
assert hasattr(always_schedule, "pk") is True
# Minute schedule
minute_schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=Schedule.MINUTES,
minutes=10,
)
assert hasattr(minute_schedule, "pk") is True
# Cron schedule
cron_schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=Schedule.CRON,
cron="0 22 * * 1-5",
)
assert hasattr(cron_schedule, "pk") is True
assert cron_schedule.full_clean() is None
assert cron_schedule.__str__() == "django_q.tests.tasks.word_multiply"
with pytest.raises(ValidationError):
create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=Schedule.CRON,
cron="0 22 * * 1-12",
)
# All other types
for t in Schedule.TYPE:
if t[0] == Schedule.CRON:
continue
schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="django",
schedule_type=t[0],
repeats=1,
hook="django_q.tests.tasks.result",
)
assert schedule is not None
assert schedule.last_run() is None
scheduler(broker=broker)
# via model
Schedule.objects.create(
func="django_q.tests.tasks.word_multiply",
args="2",
kwargs='word="django"',
schedule_type=Schedule.DAILY,
)
# scheduler
scheduler(broker=broker)
# ONCE schedule should be deleted
assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
# Catch up On
monkeypatch.setattr(Conf, "CATCH_UP", True)
now = timezone.now()
schedule = create_schedule(
"django_q.tests.tasks.word_multiply",
2,
word="catch_up",
schedule_type=Schedule.HOURLY,
next_run=timezone.now() - timedelta(hours=12),
repeats=-1,
)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run < now
# Catch up off
monkeypatch.setattr(Conf, "CATCH_UP", False)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run > now
# Done
broker.delete_queue()
monkeypatch.setattr(Conf, "PREFIX", "some_cluster_name")
# create a schedule on another cluster
schedule = create_schedule(
"math.copysign",
1,
-1,
name="test schedule on a another cluster",
hook="django_q.tests.tasks.result",
schedule_type=Schedule.HOURLY,
cluster="some_other_cluster_name",
repeats=1,
)
# run scheduler
scheduler(broker=broker)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, broker=broker)
# queue must be empty
assert task_queue.qsize() == 0
monkeypatch.setattr(Conf, "PREFIX", "default")
# create a schedule on the same cluster
schedule = create_schedule(
"math.copysign",
1,
-1,
name="test schedule with no cluster",
hook="django_q.tests.tasks.result",
schedule_type=Schedule.HOURLY,
cluster="default",
repeats=1,
)
# run scheduler
scheduler(broker=broker)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, broker=broker)
# queue must contain a task
assert task_queue.qsize() == 1
def test_scheduler(broker, monkeypatch):
broker.list_key = 'scheduler_test:q'
broker.delete_queue()
schedule = create_schedule('math.copysign',
1, -1,
name='test math',
hook='django_q.tests.tasks.result',
schedule_type=Schedule.HOURLY,
repeats=1)
assert schedule.last_run() is None
# check duplicate constraint
with pytest.raises(IntegrityError):
schedule = create_schedule('math.copysign',
1, -1,
name='test math',
hook='django_q.tests.tasks.result',
schedule_type=Schedule.HOURLY,
repeats=1)
# run scheduler
scheduler(broker=broker)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, broker=broker)
assert task_queue.qsize() == 1
assert broker.queue_size() == 0
task_queue.put('STOP')
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('b', -1))
assert result_queue.qsize() == 1
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.repeats == 0
assert schedule.last_run() is not None
assert schedule.success() is True
assert schedule.next_run < arrow.get(timezone.now()).shift(hours=+1)
task = fetch(schedule.task)
assert task is not None
assert task.success is True
assert task.result < 0
# Once schedule with delete
once_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.ONCE,
repeats=-1,
hook='django_q.tests.tasks.result'
)
assert hasattr(once_schedule, 'pk') is True
# negative repeats
always_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.DAILY,
repeats=-1,
hook='django_q.tests.tasks.result'
)
assert hasattr(always_schedule, 'pk') is True
# Minute schedule
minute_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.MINUTES,
minutes=10)
assert hasattr(minute_schedule, 'pk') is True
# Cron schedule
cron_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.CRON,
cron="0 22 * * 1-5")
assert hasattr(cron_schedule, 'pk') is True
assert cron_schedule.full_clean() is None
assert cron_schedule.__str__() == 'django_q.tests.tasks.word_multiply'
with pytest.raises(ValidationError):
create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.CRON,
cron="0 22 * * 1-12")
# All other types
for t in Schedule.TYPE:
if t[0] == Schedule.CRON:
continue
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=t[0],
repeats=1,
hook='django_q.tests.tasks.result'
)
assert schedule is not None
assert schedule.last_run() is None
scheduler(broker=broker)
# via model
Schedule.objects.create(func='django_q.tests.tasks.word_multiply',
args='2',
kwargs='word="django"',
schedule_type=Schedule.DAILY
)
# scheduler
scheduler(broker=broker)
# ONCE schedule should be deleted
assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
# Catch up On
monkeypatch.setattr(Conf, 'CATCH_UP', True)
now = timezone.now()
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='catch_up',
schedule_type=Schedule.HOURLY,
next_run=timezone.now() - timedelta(hours=12),
repeats=-1
)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run < now
# Catch up off
monkeypatch.setattr(Conf, 'CATCH_UP', False)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run > now
# Done
broker.delete_queue()
def _sync(pack):
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task['id']
def worker(id: str,
cluster_id: str,
task_queue: Queue,
result_queue: Queue,
timer: Value,
timeout: int = Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:param timeout: number of seconds wait for a worker to finish.
:type id: str
:type cluster_id: str
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(_(f"{name} ready for work at {current_process().pid}"))
# Create Worker model
model = WorkerModel.objects.create(id=id,
cluster_id=cluster_id,
pid=current_process().pid,
task=None)
task_count = 0
if timeout is None:
timeout = -1
# Start reading the task queue
for task in iter(
task_queue.get, "STOP"
): # Task should be provided as task ID which is then retrieved
result = None
timer.value = -1 # Idle
task_count += 1
# Get the function from the task
logger.info(_(f'{name} processing [{task["name"]}]'))
f = task["func"]
# if it's not an instance try to get it from the string
if not callable(task["func"]):
try:
module, func = f.rsplit(".", 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False)
if error_reporter:
error_reporter.report()
# We're still going
if not result:
close_old_django_connections()
# Set worker task details
# model.task = get_task_representation(task)
# model.save(update_fields=['task'])
timer_value = task.pop("timeout", timeout)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task["args"], **task["kwargs"])
result = (res, True)
except Exception as e:
result = (f"{e} : {traceback.format_exc()}", False)
if error_reporter:
error_reporter.report()
if task.get("sync", False):
raise
# Clear task details
# model.task = None
# model.save(update_fields=['task'])
with timer.get_lock():
# Process result
task["result"] = result[0]
task["success"] = result[1]
task["stopped"] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE or rss_check():
timer.value = -2 # Recycled
break
logger.info(_(f"{name} stopped doing work"))
def test_simple_async_report_send(rf, admin_user):
broker = get_broker()
assert broker.queue_size() == 0
request = rf.get("/")
request.query_params = {}
request.user = admin_user
report = LeaseStatisticReport()
response = report.get_response(request)
assert response.data
assert broker.queue_size() == 1
# Run async task
task_queue = Queue()
result_queue = Queue()
event = Event()
event.set()
pusher(task_queue, event, broker=broker)
assert task_queue.qsize() == 1
assert queue_size(broker=broker) == 0
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
assert task_queue.qsize() == 0
assert result_queue.qsize() == 1
result_queue.put("STOP")
monitor(result_queue)
assert result_queue.qsize() == 0
broker.delete_queue()
# Test report file have been sent via email
assert len(mail.outbox) == 1
assert len(mail.outbox[0].attachments) == 1
class Sentinel(object):
def __init__(
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.pid = current_process().pid
self.cluster_id = cluster_id
self.parent_pid = get_ppid()
self.name = current_process().name
self.broker = broker or get_broker()
self.reincarnations = 0
self.tob = timezone.now()
self.stop_event = stop_event
self.start_event = start_event
self.pool_size = Conf.WORKERS
self.pool = []
self.timeout = timeout
self.task_queue = (Queue(
maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue())
self.result_queue = Queue()
self.event_out = Event()
self.monitor = None
self.pusher = None
if start:
self.start()
def start(self):
self.broker.ping()
self.spawn_cluster()
self.guard()
def status(self):
if not self.start_event.is_set() and not self.stop_event.is_set():
return Conf.STARTING
elif self.start_event.is_set() and not self.stop_event.is_set():
if self.result_queue.empty() and self.task_queue.empty():
return Conf.IDLE
return Conf.WORKING
elif self.stop_event.is_set() and self.start_event.is_set():
if self.monitor.is_alive() or self.pusher.is_alive() or len(
self.pool) > 0:
return Conf.STOPPING
return Conf.STOPPED
def spawn_process(self, target, *args):
"""
:type target: function or class
"""
p = Process(target=target, args=args)
p.daemon = True
if target == worker:
p.daemon = Conf.DAEMONIZE_WORKERS
p.timer = args[2]
self.pool.append(p)
p.start()
return p
def spawn_pusher(self):
return self.spawn_process(pusher, self.task_queue, self.event_out,
self.broker)
def spawn_worker(self):
self.spawn_process(worker, self.task_queue, self.result_queue,
Value("f", -1), self.timeout)
def spawn_monitor(self):
return self.spawn_process(monitor, self.result_queue, self.broker)
def reincarnate(self, process):
"""
:param process: the process to reincarnate
:type process: Process or None
"""
close_old_django_connections()
if process == self.monitor:
self.monitor = self.spawn_monitor()
logger.error(
_(f"reincarnated monitor {process.name} after sudden death"))
elif process == self.pusher:
self.pusher = self.spawn_pusher()
logger.error(
_(f"reincarnated pusher {process.name} after sudden death"))
else:
self.pool.remove(process)
self.spawn_worker()
if process.timer.value == 0:
# only need to terminate on timeout, otherwise we risk destabilizing the queues
process.terminate()
logger.warn(
_(f"reincarnated worker {process.name} after timeout"))
elif int(process.timer.value) == -2:
logger.info(_(f"recycled worker {process.name}"))
else:
logger.error(
_(f"reincarnated worker {process.name} after death"))
self.reincarnations += 1
def spawn_cluster(self):
self.pool = []
Stat(self).save()
close_old_django_connections()
# spawn worker pool
for __ in range(self.pool_size):
self.spawn_worker()
# spawn auxiliary
self.monitor = self.spawn_monitor()
self.pusher = self.spawn_pusher()
# set worker cpu affinity if needed
if psutil and Conf.CPU_AFFINITY:
set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def guard(self):
logger.info(
_(f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}"
))
self.start_event.set()
Stat(self).save()
logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running."))
counter = 0
cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds
# Guard loop. Runs at least once
while not self.stop_event.is_set() or not counter:
# Check Workers
for p in self.pool:
with p.timer.get_lock():
# Are you alive?
if not p.is_alive() or p.timer.value == 0:
self.reincarnate(p)
continue
# Decrement timer if work is being done
if p.timer.value > 0:
p.timer.value -= cycle
# Check Monitor
if not self.monitor.is_alive():
self.reincarnate(self.monitor)
# Check Pusher
if not self.pusher.is_alive():
self.reincarnate(self.pusher)
# Call scheduler once a minute (or so)
counter += cycle
if counter >= 30 and Conf.SCHEDULER:
counter = 0
scheduler(broker=self.broker)
# Save current status
Stat(self).save()
sleep(cycle)
self.stop()
def stop(self):
Stat(self).save()
name = current_process().name
logger.info(_(f"{name} stopping cluster processes"))
# Stopping pusher
self.event_out.set()
# Wait for it to stop
while self.pusher.is_alive():
sleep(0.1)
Stat(self).save()
# Put poison pills in the queue
for __ in range(len(self.pool)):
self.task_queue.put("STOP")
self.task_queue.close()
# wait for the task queue to empty
self.task_queue.join_thread()
# Wait for all the workers to exit
while len(self.pool):
for p in self.pool:
if not p.is_alive():
self.pool.remove(p)
sleep(0.1)
Stat(self).save()
# Finally stop the monitor
self.result_queue.put("STOP")
self.result_queue.close()
# Wait for the result queue to empty
self.result_queue.join_thread()
logger.info(_(f"{name} waiting for the monitor."))
# Wait for everything to close or time out
count = 0
if not self.timeout:
self.timeout = 30
while self.status() == Conf.STOPPING and count < self.timeout * 10:
sleep(0.1)
Stat(self).save()
count += 1
# Final status
Stat(self).save()
def test_cluster(broker):
broker.list_key = 'cluster_test:q'
broker.delete_queue()
task = async_task('django_q.tests.tasks.count_letters', DEFAULT_WORDLIST, broker=broker)
assert broker.queue_size() == 1
task_queue = Queue()
assert task_queue.qsize() == 0
result_queue = Queue()
assert result_queue.qsize() == 0
event = Event()
event.set()
# Test push
pusher(task_queue, event, broker=broker)
assert task_queue.qsize() == 1
assert queue_size(broker=broker) == 0
# Test work
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
assert task_queue.qsize() == 0
assert result_queue.qsize() == 1
# Test monitor
result_queue.put('STOP')
monitor(result_queue)
assert result_queue.qsize() == 0
# check result
assert result(task) == 1506
broker.delete_queue()
class Sentinel(object):
def __init__(self, stop_event, start_event, broker=None, timeout=Conf.TIMEOUT, start=True):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.pid = current_process().pid
self.parent_pid = get_ppid()
self.name = current_process().name
self.broker = broker or get_broker()
self.reincarnations = 0
self.tob = timezone.now()
self.stop_event = stop_event
self.start_event = start_event
self.pool_size = Conf.WORKERS
self.pool = []
self.timeout = timeout
self.task_queue = Queue(maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue()
self.result_queue = Queue()
self.event_out = Event()
self.monitor = None
self.pusher = None
if start:
self.start()
def start(self):
self.broker.ping()
self.spawn_cluster()
self.guard()
def status(self):
if not self.start_event.is_set() and not self.stop_event.is_set():
return Conf.STARTING
elif self.start_event.is_set() and not self.stop_event.is_set():
if self.result_queue.empty() and self.task_queue.empty():
return Conf.IDLE
return Conf.WORKING
elif self.stop_event.is_set() and self.start_event.is_set():
if self.monitor.is_alive() or self.pusher.is_alive() or len(self.pool) > 0:
return Conf.STOPPING
return Conf.STOPPED
def spawn_process(self, target, *args):
"""
:type target: function or class
"""
p = Process(target=target, args=args)
p.daemon = True
if target == worker:
p.daemon = Conf.DAEMONIZE_WORKERS
p.timer = args[2]
self.pool.append(p)
p.start()
return p
def spawn_pusher(self):
return self.spawn_process(pusher, self.task_queue, self.event_out, self.broker)
def spawn_worker(self):
self.spawn_process(worker, self.task_queue, self.result_queue, Value('f', -1), self.timeout)
def spawn_monitor(self):
return self.spawn_process(monitor, self.result_queue, self.broker)
def reincarnate(self, process):
"""
:param process: the process to reincarnate
:type process: Process or None
"""
db.connections.close_all() # Close any old connections
if process == self.monitor:
self.monitor = self.spawn_monitor()
logger.error(_("reincarnated monitor {} after sudden death").format(process.name))
elif process == self.pusher:
self.pusher = self.spawn_pusher()
logger.error(_("reincarnated pusher {} after sudden death").format(process.name))
else:
self.pool.remove(process)
self.spawn_worker()
if self.timeout and int(process.timer.value) == 0:
# only need to terminate on timeout, otherwise we risk destabilizing the queues
process.terminate()
logger.warn(_("reincarnated worker {} after timeout").format(process.name))
elif int(process.timer.value) == -2:
logger.info(_("recycled worker {}").format(process.name))
else:
logger.error(_("reincarnated worker {} after death").format(process.name))
self.reincarnations += 1
def spawn_cluster(self):
self.pool = []
Stat(self).save()
db.connection.close()
# spawn worker pool
for __ in range(self.pool_size):
self.spawn_worker()
# spawn auxiliary
self.monitor = self.spawn_monitor()
self.pusher = self.spawn_pusher()
# set worker cpu affinity if needed
if psutil and Conf.CPU_AFFINITY:
set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
def guard(self):
logger.info(_('{} guarding cluster at {}').format(current_process().name, self.pid))
self.start_event.set()
Stat(self).save()
logger.info(_('Q Cluster-{} running.').format(self.parent_pid))
scheduler(broker=self.broker)
counter = 0
cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds
# Guard loop. Runs at least once
while not self.stop_event.is_set() or not counter:
# Check Workers
for p in self.pool:
# Are you alive?
if not p.is_alive() or (self.timeout and p.timer.value == 0):
self.reincarnate(p)
continue
# Decrement timer if work is being done
if self.timeout and p.timer.value > 0:
p.timer.value -= cycle
# Check Monitor
if not self.monitor.is_alive():
self.reincarnate(self.monitor)
# Check Pusher
if not self.pusher.is_alive():
self.reincarnate(self.pusher)
# Call scheduler once a minute (or so)
counter += cycle
if counter >= 30 and Conf.SCHEDULER:
counter = 0
scheduler(broker=self.broker)
# Save current status
Stat(self).save()
sleep(cycle)
self.stop()
def stop(self):
Stat(self).save()
name = current_process().name
logger.info(_('{} stopping cluster processes').format(name))
# Stopping pusher
self.event_out.set()
# Wait for it to stop
while self.pusher.is_alive():
sleep(0.1)
Stat(self).save()
# Put poison pills in the queue
for __ in range(len(self.pool)):
self.task_queue.put('STOP')
self.task_queue.close()
# wait for the task queue to empty
self.task_queue.join_thread()
# Wait for all the workers to exit
while len(self.pool):
for p in self.pool:
if not p.is_alive():
self.pool.remove(p)
sleep(0.1)
Stat(self).save()
# Finally stop the monitor
self.result_queue.put('STOP')
self.result_queue.close()
# Wait for the result queue to empty
self.result_queue.join_thread()
logger.info(_('{} waiting for the monitor.').format(name))
# Wait for everything to close or time out
count = 0
if not self.timeout:
self.timeout = 30
while self.status() == Conf.STOPPING and count < self.timeout * 10:
sleep(0.1)
Stat(self).save()
count += 1
# Final status
Stat(self).save()
def test_cached(broker):
broker.purge_queue()
broker.cache.clear()
group = 'cache_test'
# queue the tests
task_id = async_task('math.copysign', 1, -1, cached=True, broker=broker)
async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async_task('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async_task('math.popysign', 1, -1, cached=True, broker=broker, group=group)
iter_id = async_iter('math.floor', [i for i in range(10)], cached=True)
# test wait on cache
# test wait timeout
assert result(task_id, wait=10, cached=True) is None
assert fetch(task_id, wait=10, cached=True) is None
assert result_group(group, wait=10, cached=True) is None
assert result_group(group, count=2, wait=10, cached=True) is None
assert fetch_group(group, wait=10, cached=True) is None
assert fetch_group(group, count=2, wait=10, cached=True) is None
# run a single inline cluster
task_count = 17
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
monitor(result_queue)
assert result_queue.qsize() == 0
# assert results
assert result(task_id, wait=500, cached=True) == -1
assert fetch(task_id, wait=500, cached=True).result == -1
# make sure it's not in the db backend
assert fetch(task_id) is None
# assert group
assert count_group(group, cached=True) == 6
assert count_group(group, cached=True, failures=True) == 1
assert result_group(group, cached=True) == [-1, -1, -1, -1, -1]
assert len(result_group(group, cached=True, failures=True)) == 6
assert len(fetch_group(group, cached=True)) == 6
assert len(fetch_group(group, cached=True, failures=False)) == 5
delete_group(group, cached=True)
assert count_group(group, cached=True) is None
delete_cached(task_id)
assert result(task_id, cached=True) is None
assert fetch(task_id, cached=True) is None
# iter cached
assert result(iter_id) is None
assert result(iter_id, cached=True) is not None
broker.cache.clear()
class Conf:
"""
Configuration class
"""
try:
conf = settings.Q_CLUSTER
except AttributeError:
conf = {}
# Redis server configuration . Follows standard redis keywords
REDIS = conf.get("redis", {})
# Support for Django-Redis connections
DJANGO_REDIS = conf.get("django_redis", None)
# Disque broker
DISQUE_NODES = conf.get("disque_nodes", None)
# Optional Authentication
DISQUE_AUTH = conf.get("disque_auth", None)
# Optional Fast acknowledge
DISQUE_FASTACK = conf.get("disque_fastack", False)
# IronMQ broker
IRON_MQ = conf.get("iron_mq", None)
# SQS broker
SQS = conf.get("sqs", None)
# ORM broker
ORM = conf.get("orm", None)
# ORM support for read/write replicas
HAS_REPLICA = conf.get("has_replica", False)
# Custom broker class
BROKER_CLASS = conf.get("broker_class", None)
# Database Poll
POLL = conf.get("poll", 0.2)
# MongoDB broker
MONGO = conf.get("mongo", None)
MONGO_DB = conf.get("mongo_db", None)
# Name of the cluster or site. For when you run multiple sites on one redis server
PREFIX = conf.get("name", "default")
# Log output level
LOG_LEVEL = conf.get("log_level", "INFO")
# Maximum number of successful tasks kept in the database. 0 saves everything. -1 saves none
# Failures are always saved
SAVE_LIMIT = conf.get("save_limit", 250)
# Guard loop sleep in seconds. Should be between 0 and 60 seconds.
GUARD_CYCLE = conf.get("guard_cycle", 0.5)
# Disable the scheduler
SCHEDULER = conf.get("scheduler", True)
# Number of workers in the pool. Default is cpu count if implemented, otherwise 4.
WORKERS = conf.get("workers", False)
if not WORKERS:
try:
WORKERS = cpu_count()
# in rare cases this might fail
except NotImplementedError:
# try psutil
if psutil:
WORKERS = psutil.cpu_count() or 4
else:
# sensible default
WORKERS = 4
# Option to undaemonize the workers and allow them to spawn child processes
DAEMONIZE_WORKERS = conf.get("daemonize_workers", True)
# Maximum number of tasks that each cluster can work on
QUEUE_LIMIT = conf.get("queue_limit", int(WORKERS)**2)
# Sets compression of redis packages
COMPRESSED = conf.get("compress", False)
# Number of tasks each worker can handle before it gets recycled. Useful for releasing memory
RECYCLE = conf.get("recycle", 500)
# The maximum resident set size in kilobytes before a worker will recycle. Useful for limiting memory usage
# Not available on all platforms
MAX_RSS = conf.get("max_rss", None)
# Number of seconds to wait for a worker to finish.
TIMEOUT = conf.get("timeout", None)
# Whether to acknowledge unsuccessful tasks.
# This causes failed tasks to be considered delivered, thereby removing them from
# the task queue. Defaults to False.
ACK_FAILURES = conf.get("ack_failures", False)
# Number of seconds to wait for acknowledgement before retrying a task
# Only works with brokers that guarantee delivery. Defaults to 60 seconds.
RETRY = conf.get("retry", 60)
# Verify if retry and timeout settings are correct
if not TIMEOUT or (TIMEOUT > RETRY):
warn(
"""Retry and timeout are misconfigured. Set retry larger than timeout,
failure to do so will cause the tasks to be retriggered before completion.
See https://django-q.readthedocs.io/en/latest/configure.html#retry for details."""
)
# Sets the amount of tasks the cluster will try to pop off the broker.
# If it supports bulk gets.
BULK = conf.get("bulk", 1)
# The Django Admin label for this app
LABEL = conf.get("label", "Django Q")
# Sets the number of processors for each worker, defaults to all.
CPU_AFFINITY = conf.get("cpu_affinity", 0)
# Global sync option to for debugging
SYNC = conf.get("sync", False)
# The Django cache to use
CACHE = conf.get("cache", "default")
# Use the cache as result backend. Can be 'True' or an integer representing the global cache timeout.
# i.e 'cached: 60' , will make all results go the cache and expire in 60 seconds.
CACHED = conf.get("cached", False)
# If set to False the scheduler won't execute tasks in the past.
# Instead it will run once and reschedule the next run in the future. Defaults to True.
CATCH_UP = conf.get("catch_up", True)
# Use the secret key for package signing
# Django itself should raise an error if it's not configured
SECRET_KEY = settings.SECRET_KEY
# The redis stats key
Q_STAT = f"django_q:{PREFIX}:cluster"
# Optional error reporting setup
ERROR_REPORTER = conf.get("error_reporter", {})
# Optional attempt count. set to 0 for infinite attempts
MAX_ATTEMPTS = conf.get("max_attempts", 0)
# OSX doesn't implement qsize because of missing sem_getvalue()
try:
QSIZE = Queue().qsize() == 0
except (NotImplementedError, OSError):
QSIZE = False
# Getting the signal names
SIGNAL_NAMES = dict((getattr(signal, n), n) for n in dir(signal)
if n.startswith("SIG") and "_" not in n)
# Translators: Cluster status descriptions
STARTING = _("Starting")
WORKING = _("Working")
IDLE = _("Idle")
STOPPED = _("Stopped")
STOPPING = _("Stopping")
# to manage workarounds during testing
TESTING = conf.get("testing", False)
def test_scheduler(broker, monkeypatch):
broker.list_key = 'scheduler_test:q'
broker.delete_queue()
schedule = create_schedule('math.copysign',
1, -1,
name='test math',
hook='django_q.tests.tasks.result',
schedule_type=Schedule.HOURLY,
repeats=1)
assert schedule.last_run() is None
# check duplicate constraint
with pytest.raises(IntegrityError):
schedule = create_schedule('math.copysign',
1, -1,
name='test math',
hook='django_q.tests.tasks.result',
schedule_type=Schedule.HOURLY,
repeats=1)
# run scheduler
scheduler(broker=broker)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, broker=broker)
assert task_queue.qsize() == 1
assert broker.queue_size() == 0
task_queue.put('STOP')
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('b', -1))
assert result_queue.qsize() == 1
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.repeats == 0
assert schedule.last_run() is not None
assert schedule.success() is True
assert schedule.next_run < arrow.get(timezone.now()).replace(hours=+1)
task = fetch(schedule.task)
assert task is not None
assert task.success is True
assert task.result < 0
# Once schedule with delete
once_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.ONCE,
repeats=-1,
hook='django_q.tests.tasks.result'
)
assert hasattr(once_schedule, 'pk') is True
# negative repeats
always_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.DAILY,
repeats=-1,
hook='django_q.tests.tasks.result'
)
assert hasattr(always_schedule, 'pk') is True
# Minute schedule
minute_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.MINUTES,
minutes=10)
assert hasattr(minute_schedule, 'pk') is True
# All other types
for t in Schedule.TYPE:
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=t[0],
repeats=1,
hook='django_q.tests.tasks.result'
)
assert schedule is not None
assert schedule.last_run() is None
scheduler(broker=broker)
# via model
Schedule.objects.create(func='django_q.tests.tasks.word_multiply',
args='2',
kwargs='word="django"',
schedule_type=Schedule.DAILY
)
# scheduler
scheduler(broker=broker)
# ONCE schedule should be deleted
assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
# Catch up On
monkeypatch.setattr(Conf, 'CATCH_UP', True)
now = timezone.now()
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='catch_up',
schedule_type=Schedule.HOURLY,
next_run=timezone.now() - timedelta(hours=12),
repeats=-1
)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run < now
# Catch up off
monkeypatch.setattr(Conf, 'CATCH_UP', False)
scheduler(broker=broker)
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.next_run > now
# Done
broker.delete_queue()
def run_synchronously(pack):
"""Method to run a task synchoronously"""
from django_tenant_schemas_q.cluster import worker, monitor
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task["id"]
class Conf(object):
"""
Configuration class
"""
try:
conf = settings.Q_CLUSTER
except AttributeError:
conf = {}
# Redis server configuration . Follows standard redis keywords
REDIS = conf.get('redis', {})
# Support for Django-Redis connections
DJANGO_REDIS = conf.get('django_redis', None)
# Disque broker
DISQUE_NODES = conf.get('disque_nodes', None)
# Optional Authentication
DISQUE_AUTH = conf.get('disque_auth', None)
# Optional Fast acknowledge
DISQUE_FASTACK = conf.get('disque_fastack', False)
# IronMQ broker
IRON_MQ = conf.get('iron_mq', None)
# SQS broker
SQS = conf.get('sqs', None)
# ORM broker
ORM = conf.get('orm', None)
# Custom broker class
BROKER_CLASS = conf.get('broker_class', None)
# Database Poll
POLL = conf.get('poll', 0.2)
# MongoDB broker
MONGO = conf.get('mongo', None)
MONGO_DB = conf.get('mongo_db', None)
# Name of the cluster or site. For when you run multiple sites on one redis server
PREFIX = conf.get('name', 'default')
# Log output level
LOG_LEVEL = conf.get('log_level', 'INFO')
# Maximum number of successful tasks kept in the database. 0 saves everything. -1 saves none
# Failures are always saved
SAVE_LIMIT = conf.get('save_limit', 250)
# Guard loop sleep in seconds. Should be between 0 and 60 seconds.
GUARD_CYCLE = conf.get('guard_cycle', 0.5)
# Disable the scheduler
SCHEDULER = conf.get('scheduler', True)
# Number of workers in the pool. Default is cpu count if implemented, otherwise 4.
WORKERS = conf.get('workers', False)
if not WORKERS:
try:
WORKERS = cpu_count()
# in rare cases this might fail
except NotImplementedError:
# try psutil
if psutil:
WORKERS = psutil.cpu_count() or 4
else:
# sensible default
WORKERS = 4
# Option to undaemonize the workers and allow them to spawn child processes
DAEMONIZE_WORKERS = conf.get('daemonize_workers', True)
# Maximum number of tasks that each cluster can work on
QUEUE_LIMIT = conf.get('queue_limit', int(WORKERS) ** 2)
# Sets compression of redis packages
COMPRESSED = conf.get('compress', False)
# Number of tasks each worker can handle before it gets recycled. Useful for releasing memory
RECYCLE = conf.get('recycle', 500)
# Number of seconds to wait for a worker to finish.
TIMEOUT = conf.get('timeout', None)
# Whether to acknowledge unsuccessful tasks.
# This causes failed tasks to be considered delivered, thereby removing them from
# the task queue. Defaults to False.
ACK_FAILURES = conf.get('ack_failures', False)
# Number of seconds to wait for acknowledgement before retrying a task
# Only works with brokers that guarantee delivery. Defaults to 60 seconds.
RETRY = conf.get('retry', 60)
# Sets the amount of tasks the cluster will try to pop off the broker.
# If it supports bulk gets.
BULK = conf.get('bulk', 1)
# The Django Admin label for this app
LABEL = conf.get('label', 'Django Q')
# Sets the number of processors for each worker, defaults to all.
CPU_AFFINITY = conf.get('cpu_affinity', 0)
# Global sync option to for debugging
SYNC = conf.get('sync', False)
# The Django cache to use
CACHE = conf.get('cache', 'default')
# Use the cache as result backend. Can be 'True' or an integer representing the global cache timeout.
# i.e 'cached: 60' , will make all results go the cache and expire in 60 seconds.
CACHED = conf.get('cached', False)
# If set to False the scheduler won't execute tasks in the past.
# Instead it will run once and reschedule the next run in the future. Defaults to True.
CATCH_UP = conf.get('catch_up', True)
# Use the secret key for package signing
# Django itself should raise an error if it's not configured
SECRET_KEY = settings.SECRET_KEY
# The redis stats key
Q_STAT = 'django_q:{}:cluster'.format(PREFIX)
# Optional error reporting setup
ERROR_REPORTER = conf.get('error_reporter', {})
# OSX doesn't implement qsize because of missing sem_getvalue()
try:
QSIZE = Queue().qsize() == 0
except NotImplementedError:
QSIZE = False
# Getting the signal names
SIGNAL_NAMES = dict((getattr(signal, n), n) for n in dir(signal) if n.startswith('SIG') and '_' not in n)
# Translators: Cluster status descriptions
STARTING = _('Starting')
WORKING = _('Working')
IDLE = _("Idle")
STOPPED = _('Stopped')
STOPPING = _('Stopping')
# to manage workarounds during testing
TESTING = conf.get('testing', False)
