def test_login_by_email_positive(self): # when response = self.client.post(reverse('email_login'), data={'email_or_login': self.new_user.email, }) # then self.assertContains(response=response, text="Вам отправлен код!", status_code=200) issued_code = Code.objects.filter(recipient=self.new_user.email).get() self.assertIsNotNone(issued_code) # check email was sent packages = self.broker.dequeue() task_signed = packages[0][1] task = SignedPackage.loads(task_signed) self.assertEqual(task['func'].__name__, 'send_auth_email') self.assertEqual(task['args'][0].id, self.new_user.id) self.assertEqual(task['args'][1].id, issued_code.id) # check notify wast sent packages = self.broker.dequeue() task_signed = packages[0][1] task = SignedPackage.loads(task_signed) self.assertEqual(task['func'].__name__, 'notify_user_auth') self.assertEqual(task['args'][0].id, self.new_user.id) self.assertEqual(task['args'][1].id, issued_code.id) # it's not yet authorised, only code was sent self.assertFalse(self.client.is_authorised())
def save_cached(task, broker): task_key = f'{broker.list_key}:{task["id"]}' timeout = task["cached"] if timeout is True: timeout = None try: group = task.get("group", None) iter_count = task.get("iter_count", 0) # if it's a group append to the group list if group: group_key = f"{broker.list_key}:{group}:keys" group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = f"{broker.list_key}:{group}:args" # collate the results into a Task result results = [ SignedPackage.loads(broker.cache.get(k))["result"] for k in group_list ] results.append(task["result"]) task["result"] = results task["id"] = group task["args"] = SignedPackage.loads( broker.cache.get(group_args)) task.pop("iter_count", None) task.pop("group", None) if task.get("iter_cached", None): task["cached"] = task.pop("iter_cached", None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async_task next in a chain if task.get("chain", None): django_q.tasks.async_chain( task["chain"], group=group, cached=task["cached"], sync=task["sync"], broker=broker, ) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def save_cached(task, broker): task_key = '{}:{}'.format(broker.list_key, task['id']) timeout = task['cached'] if timeout is True: timeout = None try: group = task.get('group', None) iter_count = task.get('iter_count', 0) # if it's a group append to the group list if group: group_key = '{}:{}:keys'.format(broker.list_key, group) group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = '{}:{}:args'.format(broker.list_key, group) # collate the results into a Task result results = [ SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list ] results.append(task['result']) task['result'] = results task['id'] = group task['args'] = SignedPackage.loads( broker.cache.get(group_args)) task.pop('iter_count', None) task.pop('group', None) if task.get('iter_cached', None): task['cached'] = task.pop('iter_cached', None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async_task next in a chain if task.get('chain', None): tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None): """ Return a list of Tasks for a task group in the cache backend """ if not broker: broker = get_broker() start = time() if count: while True: if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait >= 0: break sleep(0.01) while True: group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id)) if group_list: task_list = [] for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if task['success'] or failures: t = Task(id=task['id'], name=task['name'], func=task['func'], hook=task.get('hook'), args=task['args'], kwargs=task['kwargs'], started=task['started'], stopped=task['stopped'], result=task['result'], group=task.get('group'), success=task['success']) task_list.append(t) return task_list if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None): """ Return the processed task from the cache backend """ if not broker: broker = get_broker() start = time() while True: r = broker.cache.get('{}:{}'.format(broker.list_key, task_id)) if r: task = SignedPackage.loads(r) t = Task(id=task['id'], name=task['name'], func=task['func'], hook=task.get('hook'), args=task['args'], kwargs=task['kwargs'], started=task['started'], stopped=task['stopped'], result=task['result'], success=task['success']) return t if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None): """ Return a list of Tasks for a task group in the cache backend """ if not broker: broker = get_broker() start = time() if count: while True: if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait >= 0: break sleep(0.01) while True: group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id)) if group_list: task_list = [] for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if task['success'] or failures: t = Task(id=task['id'], name=task['name'], func=task['func'], hook=task.get('hook'), args=task['args'], kwargs=task['kwargs'], started=task['started'], stopped=task['stopped'], result=task['result'], group=task.get('group'), success=task['success']) task_list.append(t) return task_list if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def pusher(task_queue, event, broker=None): """ Pulls tasks of the broker and puts them in the task queue :type task_queue: multiprocessing.Queue :type event: multiprocessing.Event """ if not broker: broker = get_broker() logger.info(_('{} pushing tasks at {}').format(current_process().name, current_process().pid)) while True: try: task_set = broker.dequeue() except Exception as e: logger.error(e) # broker probably crashed. Let the sentinel handle it. sleep(10) break if task_set: for task in task_set: ack_id = task[0] # unpack the task try: task = SignedPackage.loads(task[1]) except (TypeError, BadSignature) as e: logger.error(e) broker.fail(ack_id) continue task['ack_id'] = ack_id task_queue.put(task) logger.debug(_('queueing from {}').format(broker.list_key)) if event.is_set(): break logger.info(_("{} stopped pushing tasks").format(current_process().name))
def pusher(task_queue, event, broker=None): """ Pulls tasks of the broker and puts them in the task queue :type task_queue: multiprocessing.Queue :type event: multiprocessing.Event """ if not broker: broker = get_broker() logger.info( _(f"{current_process().name} pushing tasks at {current_process().pid}") ) while True: try: task_set = broker.dequeue() except Exception as e: logger.error(e, traceback.format_exc()) # broker probably crashed. Let the sentinel handle it. sleep(10) break if task_set: for task in task_set: ack_id = task[0] # unpack the task try: task = SignedPackage.loads(task[1]) except (TypeError, BadSignature) as e: logger.error(e, traceback.format_exc()) broker.fail(ack_id) continue task["ack_id"] = ack_id task_queue.put(task) logger.debug(_(f"queueing from {broker.list_key}")) if event.is_set(): break logger.info(_(f"{current_process().name} stopped pushing tasks"))
def result_group_cached(group_id, failures=False, wait=0, count=None, broker=None): """ Return a list of results for a task group from the cache backend """ if not broker: broker = get_broker() start = time() if count: while True: if count_group_cached(group_id) == count or wait and ( time() - start) * 1000 >= wait > 0: break sleep(0.01) while True: group_list = broker.cache.get('{}:{}:keys'.format( broker.list_key, group_id)) if group_list: result_list = [] for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if task['success'] or failures: result_list.append(task['result']) return result_list if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None): """ Return the processed task from the cache backend """ if not broker: broker = get_broker() start = time() while True: r = broker.cache.get('{}:{}'.format(broker.list_key, task_id)) if r: task = SignedPackage.loads(r) t = Task(id=task['id'], name=task['name'], func=task['func'], hook=task.get('hook'), args=task['args'], kwargs=task['kwargs'], started=task['started'], stopped=task['stopped'], result=task['result'], success=task['success']) return t if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def fetch_cached(task_id, wait=0, broker=None): """ Return the processed task from the cache backend """ if not broker: broker = get_broker() start = time() while True: r = broker.cache.get(f"{broker.list_key}:{task_id}") if r: task = SignedPackage.loads(r) return Task( id=task["id"], name=task["name"], func=task["func"], hook=task.get("hook"), args=task["args"], kwargs=task["kwargs"], started=task["started"], stopped=task["stopped"], result=task["result"], success=task["success"], ) if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def save_cached(task, broker): task_key = '{}:{}'.format(broker.list_key, task['id']) timeout = task['cached'] if timeout is True: timeout = None try: group = task.get('group', None) iter_count = task.get('iter_count', 0) # if it's a group append to the group list if group: group_key = '{}:{}:keys'.format(broker.list_key, group) group_list = broker.cache.get(group_key) or [] # if it's an iter group, check if we are ready if iter_count and len(group_list) == iter_count - 1: group_args = '{}:{}:args'.format(broker.list_key, group) # collate the results into a Task result results = [SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list] results.append(task['result']) task['result'] = results task['id'] = group task['args'] = SignedPackage.loads(broker.cache.get(group_args)) task.pop('iter_count', None) task.pop('group', None) if task.get('iter_cached', None): task['cached'] = task.pop('iter_cached', None) save_cached(task, broker=broker) else: save_task(task, broker) broker.cache.delete_many(group_list) broker.cache.delete_many([group_key, group_args]) return # save the group list group_list.append(task_key) broker.cache.set(group_key, group_list, timeout) # async next in a chain if task.get('chain', None): tasks.async_chain(task['chain'], group=group, cached=task['cached'], sync=task['sync'], broker=broker) # save the task broker.cache.set(task_key, SignedPackage.dumps(task), timeout) except Exception as e: logger.error(e)
def result_cached(task_id, wait=0, broker=None): """ Return the result from the cache backend """ if not broker: broker = get_broker() start = time() while True: r = broker.cache.get('{}:{}'.format(broker.list_key, task_id)) if r: return SignedPackage.loads(r)['result'] if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def _sync(pack): # Python 2.6 is unable to handle this import on top of the file # because it creates a circular dependency between tasks and cluster from django_q.cluster import worker, monitor """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) return task['id']
def result_cached(task_id, wait=0, broker=None): """ Return the result from the cache backend """ if not broker: broker = get_broker() start = time() while True: r = broker.cache.get('{}:{}'.format(broker.list_key, task_id)) if r: return SignedPackage.loads(r)['result'] if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def get(pid, cluster_id, broker=None): """ gets the current status for the cluster :param cluster_id: id of the cluster :return: Stat or Status """ if not broker: broker = get_broker() pack = broker.get_stat(Stat.get_key(cluster_id)) if pack: try: return SignedPackage.loads(pack) except BadSignature: return None return Status(pid=pid, cluster_id=cluster_id)
def get(cluster_id, broker=None): """ gets the current status for the cluster :param cluster_id: id of the cluster :return: Stat or Status """ if not broker: broker = get_broker() pack = broker.get_stat(Stat.get_key(cluster_id)) if pack: try: return SignedPackage.loads(pack) except BadSignature: return None return Status(cluster_id)
def _sync(pack): """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task['id']
def _sync(pack): """Simulate a package travelling through the cluster.""" task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put('STOP') worker(task_queue, result_queue, Value('f', -1)) result_queue.put('STOP') monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task['id']
def get_all(broker=None): """ Get the status for all currently running clusters with the same prefix and secret key. :return: list of type Stat """ if not broker: broker = get_broker() stats = [] packs = broker.get_stats(f"{Conf.Q_STAT}:*") or [] for pack in packs: try: stats.append(SignedPackage.loads(pack)) except BadSignature: continue return stats
def count_group_cached(group_id, failures=False, broker=None): """ Count the results in a group in the cache backend """ if not broker: broker = get_broker() group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id)) if group_list: if not failures: return len(group_list) failure_count = 0 for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if not task['success']: failure_count += 1 return failure_count
def get_all(broker=None): """ Get the status for all currently running clusters with the same prefix and secret key. :return: list of type Stat """ if not broker: broker = get_broker() stats = [] packs = broker.get_stats('{}:*'.format(Conf.Q_STAT)) or [] for pack in packs: try: stats.append(SignedPackage.loads(pack)) except BadSignature: continue return stats
def count_group_cached(group_id, failures=False, broker=None): """ Count the results in a group in the cache backend """ if not broker: broker = get_broker() group_list = broker.cache.get(f"{broker.list_key}:{group_id}:keys") if group_list: if not failures: return len(group_list) failure_count = 0 for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if not task["success"]: failure_count += 1 return failure_count
def get(pid: int, cluster_id: str, broker: Broker = None) -> Union[Status, None]: """ gets the current status for the cluster :param pid: :param broker: an optional broker instance :param cluster_id: id of the cluster :return: Stat or Status """ if not broker: broker = get_broker() pack = broker.get_stat(Stat.get_key(cluster_id)) if pack: try: return SignedPackage.loads(pack) except BadSignature: return None return Status(pid=pid, cluster_id=cluster_id)
def _sync(pack): """Simulate a package travelling through the cluster.""" from django_q.cluster import monitor, worker task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put("STOP") worker(task_queue, result_queue, Value("f", -1)) result_queue.put("STOP") monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task["id"]
def run_synchronously(pack): """Method to run a task synchoronously""" from django_tenant_schemas_q.cluster import worker, monitor task_queue = Queue() result_queue = Queue() task = SignedPackage.loads(pack) task_queue.put(task) task_queue.put("STOP") worker(task_queue, result_queue, Value("f", -1)) result_queue.put("STOP") monitor(result_queue) task_queue.close() task_queue.join_thread() result_queue.close() result_queue.join_thread() return task["id"]
def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None): """ Return a list of Tasks for a task group in the cache backend """ if not broker: broker = get_broker() start = time() if count: while True: if (count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait >= 0): break sleep(0.01) while True: group_list = broker.cache.get(f"{broker.list_key}:{group_id}:keys") if group_list: task_list = [] for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if task["success"] or failures: t = Task( id=task["id"], name=task["name"], func=task["func"], hook=task.get("hook"), args=task["args"], kwargs=task["kwargs"], started=task["started"], stopped=task["stopped"], result=task["result"], group=task.get("group"), success=task["success"], ) task_list.append(t) return task_list if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def result_group_cached(group_id, failures=False, wait=0, count=None, broker=None): """ Return a list of results for a task group from the cache backend """ if not broker: broker = get_broker() start = time() if count: while True: if count_group_cached(group_id) == count or wait and (time() - start) * 1000 >= wait > 0: break sleep(0.01) while True: group_list = broker.cache.get('{}:{}:keys'.format(broker.list_key, group_id)) if group_list: result_list = [] for task_key in group_list: task = SignedPackage.loads(broker.cache.get(task_key)) if task['success'] or failures: result_list.append(task['result']) return result_list if (time() - start) * 1000 >= wait >= 0: break sleep(0.01)
def task(self): return SignedPackage.loads(self.payload)
def task(self): return SignedPackage.loads(self.payload)