def test_login_by_email_positive(self):
# when
response = self.client.post(reverse('email_login'),
data={'email_or_login': self.new_user.email, })
# then
self.assertContains(response=response, text="Вам отправлен код!", status_code=200)
issued_code = Code.objects.filter(recipient=self.new_user.email).get()
self.assertIsNotNone(issued_code)
# check email was sent
packages = self.broker.dequeue()
task_signed = packages[0][1]
task = SignedPackage.loads(task_signed)
self.assertEqual(task['func'].__name__, 'send_auth_email')
self.assertEqual(task['args'][0].id, self.new_user.id)
self.assertEqual(task['args'][1].id, issued_code.id)
# check notify wast sent
packages = self.broker.dequeue()
task_signed = packages[0][1]
task = SignedPackage.loads(task_signed)
self.assertEqual(task['func'].__name__, 'notify_user_auth')
self.assertEqual(task['args'][0].id, self.new_user.id)
self.assertEqual(task['args'][1].id, issued_code.id)
# it's not yet authorised, only code was sent
self.assertFalse(self.client.is_authorised())
def save_cached(task, broker):
task_key = f'{broker.list_key}:{task["id"]}'
timeout = task["cached"]
if timeout is True:
timeout = None
try:
group = task.get("group", None)
iter_count = task.get("iter_count", 0)
# if it's a group append to the group list
if group:
group_key = f"{broker.list_key}:{group}:keys"
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = f"{broker.list_key}:{group}:args"
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))["result"]
for k in group_list
]
results.append(task["result"])
task["result"] = results
task["id"] = group
task["args"] = SignedPackage.loads(
broker.cache.get(group_args))
task.pop("iter_count", None)
task.pop("group", None)
if task.get("iter_cached", None):
task["cached"] = task.pop("iter_cached", None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get("chain", None):
django_q.tasks.async_chain(
task["chain"],
group=group,
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception as e:
logger.error(e)
def save_cached(task, broker):
task_key = '{}:{}'.format(broker.list_key, task['id'])
timeout = task['cached']
if timeout is True:
timeout = None
try:
group = task.get('group', None)
iter_count = task.get('iter_count', 0)
# if it's a group append to the group list
if group:
group_key = '{}:{}:keys'.format(broker.list_key, group)
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = '{}:{}:args'.format(broker.list_key, group)
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))['result']
for k in group_list
]
results.append(task['result'])
task['result'] = results
task['id'] = group
task['args'] = SignedPackage.loads(
broker.cache.get(group_args))
task.pop('iter_count', None)
task.pop('group', None)
if task.get('iter_cached', None):
task['cached'] = task.pop('iter_cached', None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get('chain', None):
tasks.async_chain(task['chain'],
group=group,
cached=task['cached'],
sync=task['sync'],
broker=broker)
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception as e:
logger.error(e)
def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None):
"""
Return a list of Tasks for a task group in the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
while True:
group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id))
if group_list:
task_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task['success'] or failures:
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
group=task.get('group'),
success=task['success'])
task_list.append(t)
return task_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None):
"""
Return the processed task from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get('{}:{}'.format(broker.list_key, task_id))
if r:
task = SignedPackage.loads(r)
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
success=task['success'])
return t
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None):
"""
Return a list of Tasks for a task group in the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
while True:
group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id))
if group_list:
task_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task['success'] or failures:
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
group=task.get('group'),
success=task['success'])
task_list.append(t)
return task_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def pusher(task_queue, event, broker=None):
"""
Pulls tasks of the broker and puts them in the task queue
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
if not broker:
broker = get_broker()
logger.info(_('{} pushing tasks at {}').format(current_process().name, current_process().pid))
while True:
try:
task_set = broker.dequeue()
except Exception as e:
logger.error(e)
# broker probably crashed. Let the sentinel handle it.
sleep(10)
break
if task_set:
for task in task_set:
ack_id = task[0]
# unpack the task
try:
task = SignedPackage.loads(task[1])
except (TypeError, BadSignature) as e:
logger.error(e)
broker.fail(ack_id)
continue
task['ack_id'] = ack_id
task_queue.put(task)
logger.debug(_('queueing from {}').format(broker.list_key))
if event.is_set():
break
logger.info(_("{} stopped pushing tasks").format(current_process().name))
def pusher(task_queue, event, broker=None):
"""
Pulls tasks of the broker and puts them in the task queue
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
if not broker:
broker = get_broker()
logger.info(
_(f"{current_process().name} pushing tasks at {current_process().pid}")
)
while True:
try:
task_set = broker.dequeue()
except Exception as e:
logger.error(e, traceback.format_exc())
# broker probably crashed. Let the sentinel handle it.
sleep(10)
break
if task_set:
for task in task_set:
ack_id = task[0]
# unpack the task
try:
task = SignedPackage.loads(task[1])
except (TypeError, BadSignature) as e:
logger.error(e, traceback.format_exc())
broker.fail(ack_id)
continue
task["ack_id"] = ack_id
task_queue.put(task)
logger.debug(_(f"queueing from {broker.list_key}"))
if event.is_set():
break
logger.info(_(f"{current_process().name} stopped pushing tasks"))
def result_group_cached(group_id,
failures=False,
wait=0,
count=None,
broker=None):
"""
Return a list of results for a task group from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if count_group_cached(group_id) == count or wait and (
time() - start) * 1000 >= wait > 0:
break
sleep(0.01)
while True:
group_list = broker.cache.get('{}:{}:keys'.format(
broker.list_key, group_id))
if group_list:
result_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task['success'] or failures:
result_list.append(task['result'])
return result_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None):
"""
Return the processed task from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get('{}:{}'.format(broker.list_key, task_id))
if r:
task = SignedPackage.loads(r)
t = Task(id=task['id'],
name=task['name'],
func=task['func'],
hook=task.get('hook'),
args=task['args'],
kwargs=task['kwargs'],
started=task['started'],
stopped=task['stopped'],
result=task['result'],
success=task['success'])
return t
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None):
"""
Return the processed task from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get(f"{broker.list_key}:{task_id}")
if r:
task = SignedPackage.loads(r)
return Task(
id=task["id"],
name=task["name"],
func=task["func"],
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
started=task["started"],
stopped=task["stopped"],
result=task["result"],
success=task["success"],
)
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def save_cached(task, broker):
task_key = '{}:{}'.format(broker.list_key, task['id'])
timeout = task['cached']
if timeout is True:
timeout = None
try:
group = task.get('group', None)
iter_count = task.get('iter_count', 0)
# if it's a group append to the group list
if group:
group_key = '{}:{}:keys'.format(broker.list_key, group)
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = '{}:{}:args'.format(broker.list_key, group)
# collate the results into a Task result
results = [SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list]
results.append(task['result'])
task['result'] = results
task['id'] = group
task['args'] = SignedPackage.loads(broker.cache.get(group_args))
task.pop('iter_count', None)
task.pop('group', None)
if task.get('iter_cached', None):
task['cached'] = task.pop('iter_cached', None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async next in a chain
if task.get('chain', None):
tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker)
# save the task
broker.cache.set(task_key,
SignedPackage.dumps(task),
timeout)
except Exception as e:
logger.error(e)
def result_cached(task_id, wait=0, broker=None):
"""
Return the result from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get('{}:{}'.format(broker.list_key, task_id))
if r:
return SignedPackage.loads(r)['result']
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def _sync(pack):
# Python 2.6 is unable to handle this import on top of the file
# because it creates a circular dependency between tasks and cluster
from django_q.cluster import worker, monitor
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
monitor(result_queue)
return task['id']
def result_cached(task_id, wait=0, broker=None):
"""
Return the result from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
while True:
r = broker.cache.get('{}:{}'.format(broker.list_key, task_id))
if r:
return SignedPackage.loads(r)['result']
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def get(pid, cluster_id, broker=None):
"""
gets the current status for the cluster
:param cluster_id: id of the cluster
:return: Stat or Status
"""
if not broker:
broker = get_broker()
pack = broker.get_stat(Stat.get_key(cluster_id))
if pack:
try:
return SignedPackage.loads(pack)
except BadSignature:
return None
return Status(pid=pid, cluster_id=cluster_id)
def get(cluster_id, broker=None):
"""
gets the current status for the cluster
:param cluster_id: id of the cluster
:return: Stat or Status
"""
if not broker:
broker = get_broker()
pack = broker.get_stat(Stat.get_key(cluster_id))
if pack:
try:
return SignedPackage.loads(pack)
except BadSignature:
return None
return Status(cluster_id)
def _sync(pack):
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task['id']
def _sync(pack):
"""Simulate a package travelling through the cluster."""
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put('STOP')
worker(task_queue, result_queue, Value('f', -1))
result_queue.put('STOP')
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task['id']
def get_all(broker=None):
"""
Get the status for all currently running clusters with the same prefix
and secret key.
:return: list of type Stat
"""
if not broker:
broker = get_broker()
stats = []
packs = broker.get_stats(f"{Conf.Q_STAT}:*") or []
for pack in packs:
try:
stats.append(SignedPackage.loads(pack))
except BadSignature:
continue
return stats
def count_group_cached(group_id, failures=False, broker=None):
"""
Count the results in a group in the cache backend
"""
if not broker:
broker = get_broker()
group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id))
if group_list:
if not failures:
return len(group_list)
failure_count = 0
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if not task['success']:
failure_count += 1
return failure_count
def get_all(broker=None):
"""
Get the status for all currently running clusters with the same prefix
and secret key.
:return: list of type Stat
"""
if not broker:
broker = get_broker()
stats = []
packs = broker.get_stats('{}:*'.format(Conf.Q_STAT)) or []
for pack in packs:
try:
stats.append(SignedPackage.loads(pack))
except BadSignature:
continue
return stats
def count_group_cached(group_id, failures=False, broker=None):
"""
Count the results in a group in the cache backend
"""
if not broker:
broker = get_broker()
group_list = broker.cache.get(f"{broker.list_key}:{group_id}:keys")
if group_list:
if not failures:
return len(group_list)
failure_count = 0
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if not task["success"]:
failure_count += 1
return failure_count
def get(pid: int, cluster_id: str, broker: Broker = None) -> Union[Status, None]:
"""
gets the current status for the cluster
:param pid:
:param broker: an optional broker instance
:param cluster_id: id of the cluster
:return: Stat or Status
"""
if not broker:
broker = get_broker()
pack = broker.get_stat(Stat.get_key(cluster_id))
if pack:
try:
return SignedPackage.loads(pack)
except BadSignature:
return None
return Status(pid=pid, cluster_id=cluster_id)
def _sync(pack):
"""Simulate a package travelling through the cluster."""
from django_q.cluster import monitor, worker
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task["id"]
def run_synchronously(pack):
"""Method to run a task synchoronously"""
from django_tenant_schemas_q.cluster import worker, monitor
task_queue = Queue()
result_queue = Queue()
task = SignedPackage.loads(pack)
task_queue.put(task)
task_queue.put("STOP")
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue)
task_queue.close()
task_queue.join_thread()
result_queue.close()
result_queue.join_thread()
return task["id"]
def fetch_group_cached(group_id,
failures=True,
wait=0,
count=None,
broker=None):
"""
Return a list of Tasks for a task group in the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if (count_group_cached(group_id) == count
or wait and (time() - start) * 1000 >= wait >= 0):
break
sleep(0.01)
while True:
group_list = broker.cache.get(f"{broker.list_key}:{group_id}:keys")
if group_list:
task_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task["success"] or failures:
t = Task(
id=task["id"],
name=task["name"],
func=task["func"],
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
started=task["started"],
stopped=task["stopped"],
result=task["result"],
group=task.get("group"),
success=task["success"],
)
task_list.append(t)
return task_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def result_group_cached(group_id, failures=False, wait=0, count=None, broker=None):
"""
Return a list of results for a task group from the cache backend
"""
if not broker:
broker = get_broker()
start = time()
if count:
while True:
if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait > 0:
break
sleep(0.01)
while True:
group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id))
if group_list:
result_list = []
for task_key in group_list:
task = SignedPackage.loads(broker.cache.get(task_key))
if task['success'] or failures:
result_list.append(task['result'])
return result_list
if (time() - start) * 1000 >= wait >= 0:
break
sleep(0.01)
def task(self):
return SignedPackage.loads(self.payload)
def task(self):
return SignedPackage.loads(self.payload)
