def liberate_collect_emails(results, mail_path, options): """ Send off data mining tasks """ msg_tasks = [] results = results or [] for result in results: inbox = [(mail_path, result['folder'], email_id) for email_id in result['ids']] msg_tasks.extend(inbox) task_len = len(msg_tasks) if task_len > 0: msg_tasks = liberate_message.chunks(msg_tasks, 100).group() task_group_skew(msg_tasks, step=10) msg_tasks = chain( msg_tasks, liberate_convert_box.s(mail_path, options), liberate_fetch_info.s(options), liberate_tarball.s(options), liberation_finish.s(options) ) else: options["noEmails"] = True data = {"results": []} msg_tasks = chain( liberate_convert_box.s(data, mail_path, options), liberate_fetch_info.s(options), liberate_tarball.s(options), liberation_finish.s(options) ) async_result = msg_tasks.apply_async() lib_status = get_user_model().objects.get(id=options["user"]).liberation lib_status.async_result = async_result.id lib_status.save()
def batch_delete_items(model, args=None, kwargs=None, skip_items=None, limit_items=None, batch_number=500): """If something goes wrong and you've got a lot of orphaned entries in the database, then this is the task you want. Be aware: this task pulls a list of PKs from the database which may cause increased memory use in the short term. * model is a string * args and kwargs should be obvious * batch_number is the number of delete tasks that get sent off in one go """ _model = apps.get_app_config("inboxen").get_model(model) if args is None and kwargs is None: raise Exception("You need to specify some filter options!") elif args is None: args = [] elif kwargs is None: kwargs = {} items = _model.objects.only('pk').filter(*args, **kwargs) items = [(model, item.pk) for item in items.iterator()] if skip_items is not None: items = items[skip_items:] if limit_items is not None: items = items[:limit_items] if len(items) == 0: return items = delete_inboxen_item.chunks(items, batch_number).group() task_group_skew(items, step=batch_number/10.0) items.apply_async()
def batch_delete_items(model, args=None, kwargs=None, batch_number=500): """If something goes wrong and you've got a lot of orphaned entries in the database, then this is the task you want. Be aware: this task pulls a list of PKs from the database which may cause increased memory use in the short term. * model is a string * args and kwargs should be obvious * batch_number is the number of delete tasks that get sent off in one go """ _model = apps.get_app_config("inboxen").get_model(model) if args is None and kwargs is None: raise Exception("You need to specify some filter options!") elif args is None: args = [] elif kwargs is None: kwargs = {} items = _model.objects.only('pk').filter(*args, **kwargs) items = [(model, item.pk) for item in items.iterator()] if len(items) == 0: return items = delete_inboxen_item.chunks(items, batch_number).group() task_group_skew(items, step=batch_number / 10.0) items.apply_async()
def batch_delete_items(model, args=None, kwargs=None, skip_items=None, limit_items=None, batch_number=500, chunk_size=10000, delay=20): """If something goes wrong and you've got a lot of orphaned entries in the database, then this is the task you want. Be aware: this task pulls a list of PKs from the database which may cause increased memory use in the short term. * model is a string * args and kwargs should be obvious * batch_number is the number of delete tasks that get sent off in one go * chunk_size is the number of PKs that are loaded into memory at once * delay is the number of seconds between each batch of delete tasks """ items = create_queryset(model, args=args, kwargs=kwargs, skip_items=skip_items, limit_items=limit_items) for idx, chunk in chunk_queryset(items, chunk_size): items = delete_inboxen_item.chunks([(model, i) for i in chunk], batch_number).group() task_group_skew(items, start=(idx + 1) * delay, step=delay) items.apply_async()
def user_suspended_delete_user(kwargs, batch_number=500, chunk_size=10000, delay=20): users = create_queryset(get_user_model(), kwargs=kwargs) for idx, chunk in chunk_queryset(users, chunk_size): user_tasks = delete_account.chunks([(i, ) for i in chunk], batch_number).group() task_group_skew(user_tasks, start=idx + 1, step=delay) user_tasks.delay()
def calculate_quota(batch_number=500, chunk_size=10000, delay=20): if not settings.PER_USER_EMAIL_QUOTA: return users = get_user_model().objects.all() for idx, chunk in chunk_queryset(users, chunk_size): user_tasks = calculate_user_quota.chunks([(i, ) for i in chunk], batch_number).group() task_group_skew(user_tasks, start=(idx + 1) * delay, step=delay) user_tasks.delay()
def user_suspended_delete_emails(kwargs, batch_number=500, chunk_size=10000, delay=20): kwargs = {"inbox__user__%s" % k: v for k, v in kwargs.items()} emails = create_queryset("email", kwargs=kwargs) for idx, chunk in chunk_queryset(emails, chunk_size): email_tasks = delete_inboxen_item.chunks([("email", i) for i in chunk], batch_number).group() task_group_skew(email_tasks, start=(idx + 1) * delay, step=delay) email_tasks.delay()
def calculate_quota(batch_number=500): if not settings.PER_USER_EMAIL_QUOTA: return users = get_user_model().objects.only("pk") user_tasks = calculate_user_quota.chunks([(i.pk,) for i in users.iterator()], batch_number).group() if len(user_tasks) == 0: return task_group_skew(user_tasks, step=batch_number/10.0) user_tasks.delay()
def post(self, *args, **kwargs): qs = self.get_queryset() # this is kinda bad, but nested forms aren't supported in all browsers if "delete-single" in self.request.POST: try: email_id = int(self.request.POST["delete-single"], 16) email = qs.get(id=email_id) except (self.model.DoesNotExist, ValueError): raise Http404 email.delete() return HttpResponseRedirect(self.get_success_url()) elif "important-single" in self.request.POST: try: email_id = int(self.request.POST["important-single"], 16) email = qs.get(id=email_id) except (self.model.DoesNotExist, ValueError): raise Http404 email.important = not email.important email.save(update_fields=["important"]) return HttpResponseRedirect(self.get_success_url()) emails = [] for email in self.request.POST: if self.request.POST[email] == "email": try: email_id = int(email, 16) emails.append(email_id) except ValueError: raise Http404 if len(emails) == 0: # nothing was selected, return early return HttpResponseRedirect(self.get_success_url()) qs = qs.filter(id__in=emails).order_by("id") if "unimportant" in self.request.POST: qs.update(important=False) elif "important" in self.request.POST: qs.update(important=True) elif "delete" in self.request.POST: email_ids = [("email", email.id) for email in qs] qs.update(deleted=True) delete_task = delete_inboxen_item.chunks(email_ids, 500).group() task_group_skew(delete_task, step=50) delete_task.apply_async() return HttpResponseRedirect(self.get_success_url())
def calculate_quota(batch_number=500): if not settings.PER_USER_EMAIL_QUOTA: return users = get_user_model().objects.only("pk") user_tasks = calculate_user_quota.chunks([(i.pk, ) for i in users.iterator()], batch_number).group() if len(user_tasks) == 0: return task_group_skew(user_tasks, step=batch_number / 10.0) user_tasks.delay()
def batch_set_new_flags(user_id=None, args=None, kwargs=None, batch_number=500): inbox_list = create_queryset("inbox", args=args, kwargs=kwargs).distinct().values_list( "pk", "user_id") inboxes = [] users = set() for inbox, user in inbox_list.iterator(): inboxes.append((user, inbox)) users.add((user, )) inbox_tasks = inbox_new_flag.chunks(inboxes, batch_number).group() task_group_skew(inbox_tasks, step=batch_number / 10.0) inbox_tasks.apply_async() if user_id is None and users: user_tasks = inbox_new_flag.chunks(users, batch_number).group() task_group_skew(user_tasks, step=batch_number / 10.0) user_tasks.apply_async() elif user_id is not None: inbox_new_flag.delay(user_id)