def handle(self, *args, **options):
profile_clear = {
'last_login_ip': '',
'email': None,
'fxa_id': None,
}
one_day_ago = datetime.now() - timedelta(days=1)
seven_years_ago = datetime.now() - timedelta(days=365 * 7)
seven_year_q = Q(modified__lt=seven_years_ago)
one_day_q = Q(~Q(**profile_clear),
addons=None,
modified__lt=one_day_ago,
banned=None)
users = list(
UserProfile.objects.filter(seven_year_q | one_day_q, deleted=True))
addons_qs = Addon.unfiltered.filter(status=amo.STATUS_DELETED,
authors__in=users)
addon_ids = list(addons_qs.values_list('id', flat=True))
log.info('Clearing %s for %d users', profile_clear.keys(), len(users))
for user in users:
user.update(**profile_clear, _signal=False)
if addon_ids:
delete_addons.delay(addon_ids, with_deleted=True)
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.info('Collecting data to delete')
logs = (
ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP)
.values_list('id', flat=True)
)
collections_to_delete = Collection.objects.filter(
created__lt=days_ago(2), type=amo.COLLECTION_ANONYMOUS
).values_list('id', flat=True)
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
for chunk in chunked(collections_to_delete, 100):
tasks.delete_anonymous_collections.delay(chunk)
two_weeks_ago = days_ago(15)
# Delete stale add-ons with no versions. Should soft-delete add-ons that
# are somehow not in incomplete status, hard-delete the rest. No email
# should be sent in either case.
versionless_addons = Addon.objects.filter(
versions__pk=None, created__lte=two_weeks_ago
).values_list('pk', flat=True)
for chunk in chunked(versionless_addons, 100):
delete_addons.delay(chunk)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(created__lte=two_weeks_ago).order_by('id')
for file_upload in stale_uploads:
log.info(
'[FileUpload:{uuid}] Removing file: {path}'.format(
uuid=file_upload.uuid, path=file_upload.path
)
)
if file_upload.path:
try:
storage.delete(file_upload.path)
except OSError:
pass
file_upload.delete()
# Delete stale ScannerResults.
ScannerResult.objects.filter(upload=None, version=None).delete()
def gc(test_result=True):
"""Site-wide garbage collections."""
def days_ago(days):
return datetime.today() - timedelta(days=days)
log.info('Collecting data to delete')
logs = (
ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP)
.values_list('id', flat=True)
)
for chunk in chunked(logs, 100):
tasks.delete_logs.delay(chunk)
two_weeks_ago = days_ago(15)
# Hard-delete stale add-ons with no versions. No email should be sent.
versionless_addons = Addon.unfiltered.filter(
versions__pk=None, created__lte=two_weeks_ago
).values_list('pk', flat=True)
for chunk in chunked(versionless_addons, 100):
delete_addons.delay(chunk, with_deleted=True)
# Delete stale FileUploads.
stale_uploads = FileUpload.objects.filter(created__lte=two_weeks_ago).order_by('id')
for file_upload in stale_uploads:
log.info(
'[FileUpload:{uuid}] Removing file: {path}'.format(
uuid=file_upload.uuid, path=file_upload.path
)
)
if file_upload.path:
try:
storage.delete(file_upload.path)
except OSError:
pass
file_upload.delete()
# Delete stale ScannerResults.
ScannerResult.objects.filter(upload=None, version=None).delete()
# Delete fake emails older than 90 days
FakeEmail.objects.filter(created__lte=days_ago(90)).delete()
def handle(self, *args, **options):
profile_clear = {
'last_login_ip': '',
'email': None,
'fxa_id': None,
}
one_day_ago = datetime.now() - timedelta(days=1)
seven_years_ago = datetime.now() - timedelta(days=365 * 7)
seven_year_q = Q(modified__lt=seven_years_ago)
one_day_q = Q(~Q(**profile_clear),
addons=None,
modified__lt=one_day_ago,
banned=None)
users = list(
UserProfile.objects.filter(seven_year_q | one_day_q, deleted=True))
user_restrictions = UserRestrictionHistory.objects.filter(
user__in=users)
addonuser_qs = AddonUser.objects.filter(user__in=users)
addons_qs = Addon.unfiltered.filter(
status__in=(amo.STATUS_DELETED, amo.STATUS_DISABLED),
addonuser__in=addonuser_qs,
)
addon_ids = list(addons_qs.values_list('id', flat=True))
with atomic():
log.info('Clearing %s for %d users', profile_clear.keys(),
len(users))
for user in users:
user.update(**profile_clear, _signal=False)
user_restrictions.update(ip_address='', last_login_ip='')
# IPLog is only useful to link between an ip and an activity log, so we
# can delete any IPLog pointing to an activity belonging to one of the
# users we want to clear the data for.
ip_logs = IPLog.objects.filter(activity_log__user__in=users)
ip_logs.delete()
if addon_ids:
delete_addons.delay(addon_ids, with_deleted=True)
