def liberate_collect_emails(results, mail_path, options):
""" Send off data mining tasks """
msg_tasks = []
results = results or []
for result in results:
inbox = [(mail_path, result['folder'], email_id) for email_id in result['ids']]
msg_tasks.extend(inbox)
task_len = len(msg_tasks)
if task_len > 0:
msg_tasks = liberate_message.chunks(msg_tasks, 100).group()
task_group_skew(msg_tasks, step=10)
msg_tasks = chain(
msg_tasks,
liberate_convert_box.s(mail_path, options),
liberate_fetch_info.s(options),
liberate_tarball.s(options),
liberation_finish.s(options)
)
else:
options["noEmails"] = True
data = {"results": []}
msg_tasks = chain(
liberate_convert_box.s(data, mail_path, options),
liberate_fetch_info.s(options),
liberate_tarball.s(options),
liberation_finish.s(options)
)
async_result = msg_tasks.apply_async()
lib_status = get_user_model().objects.get(id=options["user"]).liberation
lib_status.async_result = async_result.id
lib_status.save()
def batch_delete_items(model, args=None, kwargs=None, skip_items=None, limit_items=None, batch_number=500):
"""If something goes wrong and you've got a lot of orphaned entries in the
database, then this is the task you want.
Be aware: this task pulls a list of PKs from the database which may cause
increased memory use in the short term.
* model is a string
* args and kwargs should be obvious
* batch_number is the number of delete tasks that get sent off in one go
"""
_model = apps.get_app_config("inboxen").get_model(model)
if args is None and kwargs is None:
raise Exception("You need to specify some filter options!")
elif args is None:
args = []
elif kwargs is None:
kwargs = {}
items = _model.objects.only('pk').filter(*args, **kwargs)
items = [(model, item.pk) for item in items.iterator()]
if skip_items is not None:
items = items[skip_items:]
if limit_items is not None:
items = items[:limit_items]
if len(items) == 0:
return
items = delete_inboxen_item.chunks(items, batch_number).group()
task_group_skew(items, step=batch_number/10.0)
items.apply_async()
def batch_delete_items(model, args=None, kwargs=None, batch_number=500):
"""If something goes wrong and you've got a lot of orphaned entries in the
database, then this is the task you want.
Be aware: this task pulls a list of PKs from the database which may cause
increased memory use in the short term.
* model is a string
* args and kwargs should be obvious
* batch_number is the number of delete tasks that get sent off in one go
"""
_model = apps.get_app_config("inboxen").get_model(model)
if args is None and kwargs is None:
raise Exception("You need to specify some filter options!")
elif args is None:
args = []
elif kwargs is None:
kwargs = {}
items = _model.objects.only('pk').filter(*args, **kwargs)
items = [(model, item.pk) for item in items.iterator()]
if len(items) == 0:
return
items = delete_inboxen_item.chunks(items, batch_number).group()
task_group_skew(items, step=batch_number / 10.0)
items.apply_async()
def batch_delete_items(model,
args=None,
kwargs=None,
skip_items=None,
limit_items=None,
batch_number=500,
chunk_size=10000,
delay=20):
"""If something goes wrong and you've got a lot of orphaned entries in the
database, then this is the task you want.
Be aware: this task pulls a list of PKs from the database which may cause
increased memory use in the short term.
* model is a string
* args and kwargs should be obvious
* batch_number is the number of delete tasks that get sent off in one go
* chunk_size is the number of PKs that are loaded into memory at once
* delay is the number of seconds between each batch of delete tasks
"""
items = create_queryset(model,
args=args,
kwargs=kwargs,
skip_items=skip_items,
limit_items=limit_items)
for idx, chunk in chunk_queryset(items, chunk_size):
items = delete_inboxen_item.chunks([(model, i) for i in chunk],
batch_number).group()
task_group_skew(items, start=(idx + 1) * delay, step=delay)
items.apply_async()
def user_suspended_delete_user(kwargs,
batch_number=500,
chunk_size=10000,
delay=20):
users = create_queryset(get_user_model(), kwargs=kwargs)
for idx, chunk in chunk_queryset(users, chunk_size):
user_tasks = delete_account.chunks([(i, ) for i in chunk],
batch_number).group()
task_group_skew(user_tasks, start=idx + 1, step=delay)
user_tasks.delay()
def calculate_quota(batch_number=500, chunk_size=10000, delay=20):
if not settings.PER_USER_EMAIL_QUOTA:
return
users = get_user_model().objects.all()
for idx, chunk in chunk_queryset(users, chunk_size):
user_tasks = calculate_user_quota.chunks([(i, ) for i in chunk],
batch_number).group()
task_group_skew(user_tasks, start=(idx + 1) * delay, step=delay)
user_tasks.delay()
def user_suspended_delete_emails(kwargs,
batch_number=500,
chunk_size=10000,
delay=20):
kwargs = {"inbox__user__%s" % k: v for k, v in kwargs.items()}
emails = create_queryset("email", kwargs=kwargs)
for idx, chunk in chunk_queryset(emails, chunk_size):
email_tasks = delete_inboxen_item.chunks([("email", i) for i in chunk],
batch_number).group()
task_group_skew(email_tasks, start=(idx + 1) * delay, step=delay)
email_tasks.delay()
def calculate_quota(batch_number=500):
if not settings.PER_USER_EMAIL_QUOTA:
return
users = get_user_model().objects.only("pk")
user_tasks = calculate_user_quota.chunks([(i.pk,) for i in users.iterator()], batch_number).group()
if len(user_tasks) == 0:
return
task_group_skew(user_tasks, step=batch_number/10.0)
user_tasks.delay()
def post(self, *args, **kwargs):
qs = self.get_queryset()
# this is kinda bad, but nested forms aren't supported in all browsers
if "delete-single" in self.request.POST:
try:
email_id = int(self.request.POST["delete-single"], 16)
email = qs.get(id=email_id)
except (self.model.DoesNotExist, ValueError):
raise Http404
email.delete()
return HttpResponseRedirect(self.get_success_url())
elif "important-single" in self.request.POST:
try:
email_id = int(self.request.POST["important-single"], 16)
email = qs.get(id=email_id)
except (self.model.DoesNotExist, ValueError):
raise Http404
email.important = not email.important
email.save(update_fields=["important"])
return HttpResponseRedirect(self.get_success_url())
emails = []
for email in self.request.POST:
if self.request.POST[email] == "email":
try:
email_id = int(email, 16)
emails.append(email_id)
except ValueError:
raise Http404
if len(emails) == 0:
# nothing was selected, return early
return HttpResponseRedirect(self.get_success_url())
qs = qs.filter(id__in=emails).order_by("id")
if "unimportant" in self.request.POST:
qs.update(important=False)
elif "important" in self.request.POST:
qs.update(important=True)
elif "delete" in self.request.POST:
email_ids = [("email", email.id) for email in qs]
qs.update(deleted=True)
delete_task = delete_inboxen_item.chunks(email_ids, 500).group()
task_group_skew(delete_task, step=50)
delete_task.apply_async()
return HttpResponseRedirect(self.get_success_url())
def calculate_quota(batch_number=500):
if not settings.PER_USER_EMAIL_QUOTA:
return
users = get_user_model().objects.only("pk")
user_tasks = calculate_user_quota.chunks([(i.pk, )
for i in users.iterator()],
batch_number).group()
if len(user_tasks) == 0:
return
task_group_skew(user_tasks, step=batch_number / 10.0)
user_tasks.delay()
def batch_set_new_flags(user_id=None,
args=None,
kwargs=None,
batch_number=500):
inbox_list = create_queryset("inbox", args=args,
kwargs=kwargs).distinct().values_list(
"pk", "user_id")
inboxes = []
users = set()
for inbox, user in inbox_list.iterator():
inboxes.append((user, inbox))
users.add((user, ))
inbox_tasks = inbox_new_flag.chunks(inboxes, batch_number).group()
task_group_skew(inbox_tasks, step=batch_number / 10.0)
inbox_tasks.apply_async()
if user_id is None and users:
user_tasks = inbox_new_flag.chunks(users, batch_number).group()
task_group_skew(user_tasks, step=batch_number / 10.0)
user_tasks.apply_async()
elif user_id is not None:
inbox_new_flag.delay(user_id)
