def _task(**kw):
# Remove any dupes. `UNIQUE` constraint introduced in migration 504.
dupes = (ADT.objects.values_list(
'addon', 'device_type').annotate(c=Count('id')).filter(c__gt=1))
for addon, device_type, total in dupes:
devices = ADT.objects.filter(addon_id=addon, device_type=device_type)
for d in devices[:total - 1]:
d.delete()
# Remove stale device types.
devices = ADT.objects.all()
for chunk in chunked(devices, 50):
for device in chunk:
try:
device.addon
except ObjectDoesNotExist:
device.delete()
# `DEVICE_MOBILE` -> `DEVICE_MOBILE` and `DEVICE_GAIA`.
devices = ADT.objects.filter(device_type=amo.DEVICE_MOBILE.id)
for chunk in chunked(devices, 50):
for device in chunk:
if amo.DEVICE_GAIA in device.addon.device_types:
continue
device.id = None
device.device_type = amo.DEVICE_GAIA.id
device.save()
device.addon.save()
def _task(**kw):
# Remove any dupes. `UNIQUE` constraint introduced in migration 504.
dupes = (ADT.objects.values_list('addon', 'device_type')
.annotate(c=Count('id')).filter(c__gt=1))
for addon, device_type, total in dupes:
devices = ADT.objects.filter(addon_id=addon, device_type=device_type)
for d in devices[:total - 1]:
d.delete()
# Remove stale device types.
devices = ADT.objects.all()
for chunk in chunked(devices, 50):
for device in chunk:
try:
device.addon
except ObjectDoesNotExist:
device.delete()
# `DEVICE_MOBILE` -> `DEVICE_MOBILE` and `DEVICE_GAIA`.
devices = ADT.objects.filter(device_type=amo.DEVICE_MOBILE.id)
for chunk in chunked(devices, 50):
for device in chunk:
if amo.DEVICE_GAIA in device.addon.device_types:
continue
device.id = None
device.device_type = amo.DEVICE_GAIA.id
device.save()
device.addon.save()
def notify(request, job):
job = get_object_or_404(ValidationJob, pk=job)
notify_form = NotifyForm(request.POST, text='validation')
if not notify_form.is_valid():
messages.error(request, notify_form)
else:
for chunk in chunked(updated_versions(job), 100):
tasks.update_maxversions.delay(chunk, job.pk,
notify_form.cleaned_data)
updated_authors = completed_version_authors(job)
for chunk in chunked(updated_authors, 100):
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
# TODO: Offload most of this work to the task?
users_addons = list(
UserProfile.objects.filter(
pk__in=chunk).filter(**_completed_versions(
job, 'addons__versions')).values_list(
'pk', 'addons__pk').distinct())
users = list(
UserProfile.objects.filter(pk__in=set(
u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache().filter(
**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
}).annotate(
errors=Sum('versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict(
(u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons, lambda k: k[0]))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
tasks.notify_compatibility.delay(users, job,
notify_form.cleaned_data)
return redirect(reverse('zadmin.validation'))
def notify(request, job):
job = get_object_or_404(ValidationJob, pk=job)
notify_form = NotifyForm(request.POST, text='validation')
if not notify_form.is_valid():
messages.error(request, notify_form)
else:
for chunk in chunked(updated_versions(job), 100):
tasks.update_maxversions.delay(chunk, job.pk,
notify_form.cleaned_data)
updated_authors = completed_version_authors(job)
for chunk in chunked(updated_authors, 100):
# There are times when you want to punch django's ORM in
# the face. This may be one of those times.
# TODO: Offload most of this work to the task?
users_addons = list(
UserProfile.objects.filter(pk__in=chunk)
.filter(**_completed_versions(job,
'addons__versions'))
.values_list('pk', 'addons__pk').distinct())
users = list(UserProfile.objects.filter(
pk__in=set(u for u, a in users_addons)))
# Annotate fails in tests when using cached results
addons = (Addon.objects.no_cache()
.filter(**{
'pk__in': set(a for u, a in users_addons),
'versions__files__'
'validation_results__validation_job': job
})
.annotate(errors=Sum(
'versions__files__validation_results__errors')))
addons = dict((a.id, a) for a in addons)
users_addons = dict((u, [addons[a] for u, a in row])
for u, row in sorted_groupby(users_addons,
lambda k: k[0]))
for u in users:
addons = users_addons[u.pk]
u.passing_addons = [a for a in addons if a.errors == 0]
u.failing_addons = [a for a in addons if a.errors > 0]
tasks.notify_compatibility.delay(users, job,
notify_form.cleaned_data)
return redirect(reverse('zadmin.validation'))
def reindex_users(index=None):
from . import tasks
ids = UserProfile.objects.values_list('id', flat=True)
taskset = [tasks.index_users.subtask(args=[chunk],
kwargs=dict(index=index))
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(taskset).apply_async()
def deliver_hotness():
"""
Calculate hotness of all add-ons.
a = avg(users this week)
b = avg(users three weeks before this week)
hotness = (a-b) / b if a > 1000 and b > 1 else 0
"""
frozen = set(f.id for f in FrozenAddon.objects.all())
all_ids = list((Addon.objects.exclude(type=amo.ADDON_PERSONA)
.values_list('id', flat=True)))
now = datetime.now()
one_week = now - timedelta(days=7)
four_weeks = now - timedelta(days=28)
for ids in chunked(all_ids, 300):
addons = Addon.objects.no_cache().filter(id__in=ids).no_transforms()
ids = [a.id for a in addons if a.id not in frozen]
qs = (UpdateCount.objects.filter(addon__in=ids)
.values_list('addon').annotate(Avg('count')))
thisweek = dict(qs.filter(date__gte=one_week))
threeweek = dict(qs.filter(date__range=(four_weeks, one_week)))
for addon in addons:
this, three = thisweek.get(addon.id, 0), threeweek.get(addon.id, 0)
if this > 1000 and three > 1:
addon.update(hotness=(this - three) / float(three))
else:
addon.update(hotness=0)
# Let the database catch its breath.
time.sleep(10)
def handle(self, *args, **kw):
qs = Version.objects.filter(version_int=None)
print 'Found %s versions that need updating' % qs.count()
with establish_connection() as conn:
for pks in chunked(list(qs.values_list('pk', flat=True)), 1000):
add_version_int.delay(pks)
print '... added to celery.'
def mkt_gc(**kw):
"""Site-wide garbage collections."""
days_ago = lambda days: datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude(
action__in=amo.LOG_KEEP).values_list('id', flat=True))
for chunk in chunked(logs, 100):
chunk.sort()
log.debug('Deleting log entries: %s' % str(chunk))
amo.tasks.delete_logs.delay(chunk)
# Delete the dump apps over 30 days.
for app in os.listdir(settings.DUMPED_APPS_PATH):
app = os.path.join(settings.DUMPED_APPS_PATH, app)
if (os.stat(app).st_mtime <
time.time() - settings.DUMPED_APPS_DAYS_DELETE):
log.debug('Deleting old tarball: {0}'.format(app))
os.remove(app)
# Delete the dumped user installs over 30 days.
tarball_path = os.path.join(settings.DUMPED_USERS_PATH, 'tarballs')
for filename in os.listdir(tarball_path):
filepath = os.path.join(tarball_path, filename)
if (os.stat(filepath).st_mtime <
time.time() - settings.DUMPED_USERS_DAYS_DELETE):
log.debug('Deleting old tarball: {0}'.format(filepath))
os.remove(filepath)
def handle(self, *args, **options):
# Avoiding loops
from amo.utils import chunked, slugify
from users.models import UserProfile
from users.tasks import _delete_users
if not args:
print "Usage: manage.py delete_users <file>"
sys.exit(1)
if not os.path.exists(args[0]):
print "File not found: %s" % args[0]
sys.exit(1)
f = open(args[0], 'r')
data = True
print "Reading %s" % args[0]
while data:
data = f.readlines(100000) # 100000 is about 13500 user ids
data = [x.strip() for x in data] # has newlines on it
print "Sending %s users to celery" % len(data)
with establish_connection() as conn:
for chunk in chunked(data, 100):
_delete_users.apply_async(args=[chunk], connection=conn)
f.close()
print "All done."
def handle(self, *args, **kwargs):
ids = []
ids.extend(
FeedCollectionMembership.objects.values_list('group', flat=True))
ids.extend(
FeedCollection.objects.values_list('name', flat=True))
ids.extend(
FeedCollection.objects.values_list('description', flat=True))
ids.extend(
FeedShelfMembership.objects.values_list('group', flat=True))
ids.extend(
FeedShelf.objects.values_list('description', flat=True))
ids.extend(
FeedShelf.objects.values_list('name', flat=True))
ids.extend(
FeedApp.objects.values_list('description', flat=True))
ids.extend(
FeedApp.objects.values_list('pullquote_text', flat=True))
ids.extend(
Version.objects.values_list('releasenotes', flat=True))
ids.extend(
Webapp.objects.values_list('description', flat=True))
ids.extend(
Webapp.objects.values_list('privacy_policy', flat=True))
ids.extend(
Geodata.objects.values_list('banner_message', flat=True))
# Filter out any None's.
ids = filter(None, ids)
for chunk in chunked(ids, 100):
update_translations.delay(chunk)
def email_devs(request):
form = DevMailerForm(request.POST or None)
preview = EmailPreviewTopic(topic='email-devs')
if preview.filter().count():
preview_csv = reverse('zadmin.email_preview_csv',
args=[preview.topic])
else:
preview_csv = None
if request.method == 'POST' and form.is_valid():
data = form.cleaned_data
qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
amo.AUTHOR_ROLE_OWNER))
.exclude(user__email=None))
if data['recipients'] in ('payments', 'desktop_apps'):
qs = qs.exclude(addon__status=amo.STATUS_DELETED)
else:
qs = qs.filter(addon__status__in=amo.LISTED_STATUSES)
if data['recipients'] == 'eula':
qs = qs.exclude(addon__eula=None)
elif data['recipients'] in ('payments',
'payments_region_enabled',
'payments_region_disabled'):
qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
qs = qs.exclude(addon__premium_type__in=(amo.ADDON_FREE,
amo.ADDON_OTHER_INAPP))
if data['recipients'] == 'payments_region_enabled':
qs = qs.filter(addon__enable_new_regions=True)
elif data['recipients'] == 'payments_region_disabled':
qs = qs.filter(addon__enable_new_regions=False)
elif data['recipients'] == 'apps':
qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
elif data['recipients'] == 'desktop_apps':
qs = (qs.filter(addon__type=amo.ADDON_WEBAPP,
addon__addondevicetype__device_type=amo.DEVICE_DESKTOP.id))
elif data['recipients'] == 'sdk':
qs = qs.exclude(addon__versions__files__jetpack_version=None)
elif data['recipients'] == 'all_extensions':
qs = qs.filter(addon__type=amo.ADDON_EXTENSION)
else:
raise NotImplementedError('If you want to support emailing other '
'types of developers, do it here!')
if data['preview_only']:
# Clear out the last batch of previewed emails.
preview.filter().delete()
total = 0
for emails in chunked(set(qs.values_list('user__email', flat=True)),
100):
total += len(emails)
tasks.admin_email.delay(emails, data['subject'], data['message'],
preview_only=data['preview_only'],
preview_topic=preview.topic)
msg = 'Emails queued for delivery: %s' % total
if data['preview_only']:
msg = '%s (for preview only, emails not sent!)' % msg
messages.success(request, msg)
return redirect('zadmin.email_devs')
return jingo.render(request, 'zadmin/email-devs.html',
dict(form=form, preview_csv=preview_csv))
def run():
for chunk in chunked(Webapp.objects.all(), 50):
for app in chunk:
try:
generate_image_assets.delay(app)
except Exception:
pass
def handle(self, *args, **options):
ids = (CommunicationNote.objects
.filter(note_type=comm.REVIEWER_COMMENT)
.values_list('id', flat=True))
for log_chunk in chunked(ids, 100):
_fix_developer_version_notes.delay(ids)
def update_perf():
cursor = connections[multidb.get_slave()].cursor()
# The baseline is where addon_id is null.
cursor.execute(
"SELECT AVG(average) FROM perf_results WHERE addon_id IS NULL")
baseline = cursor.fetchone()[0]
# The perf_results table is a mess right now, so pull out one row
# for each addon by finding the MAX(created) and then the AVG(average)
# since there are many rows with the same (addon, created).
# This scheme completely ignores app, os, and test.
cursor.execute("""
SELECT J.addon_id, AVG(average) av FROM perf_results P INNER JOIN
(SELECT addon_id, MAX(created) c FROM perf_results
GROUP BY addon_id) J
ON ((P.addon_id=J.addon_id) AND P.created=J.c)
WHERE test='ts'
GROUP BY P.addon_id
HAVING av > %s""", (baseline,))
# A bunch of (addon, perf_average) pairs.
perf = cursor.fetchall()
with establish_connection() as conn:
for chunk in chunked(perf, 25):
tasks.update_perf.apply_async(args=[baseline, chunk],
connection=conn)
cursor.close()
def email_devs(request):
form = DevMailerForm(request.POST or None)
preview = EmailPreviewTopic(topic='email-devs')
if preview.filter().count():
preview_csv = reverse('zadmin.email_preview_csv',
args=[preview.topic])
else:
preview_csv = None
if request.method == 'POST' and form.is_valid():
data = form.cleaned_data
qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
amo.AUTHOR_ROLE_OWNER))
.exclude(user__email=None))
if data['recipients'] in ('payments', 'desktop_apps'):
qs = qs.exclude(addon__status=amo.STATUS_DELETED)
else:
qs = qs.filter(addon__status__in=amo.LISTED_STATUSES)
if data['recipients'] == 'eula':
qs = qs.exclude(addon__eula=None)
elif data['recipients'] in ('payments',
'payments_region_enabled',
'payments_region_disabled'):
qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
qs = qs.exclude(addon__premium_type__in=(amo.ADDON_FREE,
amo.ADDON_OTHER_INAPP))
if data['recipients'] == 'payments_region_enabled':
qs = qs.filter(addon__enable_new_regions=True)
elif data['recipients'] == 'payments_region_disabled':
qs = qs.filter(addon__enable_new_regions=False)
elif data['recipients'] == 'apps':
qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
elif data['recipients'] == 'desktop_apps':
qs = (qs.filter(addon__type=amo.ADDON_WEBAPP,
addon__addondevicetype__device_type=amo.DEVICE_DESKTOP.id))
elif data['recipients'] == 'sdk':
qs = qs.exclude(addon__versions__files__jetpack_version=None)
elif data['recipients'] == 'all_extensions':
qs = qs.filter(addon__type=amo.ADDON_EXTENSION)
else:
raise NotImplementedError('If you want to support emailing other '
'types of developers, do it here!')
if data['preview_only']:
# Clear out the last batch of previewed emails.
preview.filter().delete()
total = 0
for emails in chunked(set(qs.values_list('user__email', flat=True)),
100):
total += len(emails)
tasks.admin_email.delay(emails, data['subject'], data['message'],
preview_only=data['preview_only'],
preview_topic=preview.topic)
msg = 'Emails queued for delivery: %s' % total
if data['preview_only']:
msg = '%s (for preview only, emails not sent!)' % msg
messages.success(request, msg)
return redirect('zadmin.email_devs')
return jingo.render(request, 'zadmin/email-devs.html',
dict(form=form, preview_csv=preview_csv))
def update_user_ratings():
"""Update add-on author's ratings."""
cursor = connections[multidb.get_slave()].cursor()
# We build this query ahead of time because the cursor complains about data
# truncation if it does the parameters. Also, this query is surprisingly
# quick, <1sec for 6100 rows returned
q = """ SELECT
addons_users.user_id as user_id,
AVG(rating) as avg_rating
FROM reviews
INNER JOIN versions
INNER JOIN addons_users
INNER JOIN addons
ON reviews.version_id = versions.id
AND addons.id = versions.addon_id
AND addons_users.addon_id = addons.id
WHERE reviews.reply_to IS NULL
AND reviews.rating > 0
AND addons.status IN (%s)
GROUP BY addons_users.user_id
""" % (",".join(map(str, VALID_STATUSES)))
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
with establish_connection() as conn:
for chunk in chunked(d, 1000):
_update_user_ratings.apply_async(args=[chunk], connection=conn)
def handle(self, *args, **options):
task = tasks.get(options.get('task'))
if not task:
raise CommandError('Unknown task provided. Options are: %s' %
', '.join(tasks.keys()))
pks = (Webapp.objects.filter(*task['qs']).values_list(
'pk', flat=True).order_by('-last_updated'))
if 'pre' in task:
# This is run in process to ensure its run before the tasks.
pks = task['pre'](pks)
if pks:
kw = task.get('kwargs', {})
# All the remaining tasks go in one group.
grouping = []
for chunk in chunked(pks, 100):
grouping.append(task['method'].subtask(args=[chunk],
kwargs=kw))
# Add the post task on to the end.
post = None
if 'post' in task:
post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
ts = chord(grouping, post)
else:
ts = group(grouping)
ts.apply_async()
def addon_reviews_ratings():
"""Update all add-on total_reviews and average/bayesian ratings."""
addons = Addon.objects.values_list('id', flat=True)
with establish_connection() as conn:
for chunk in chunked(addons, 100):
tasks.cron_review_aggregate.apply_async(args=chunk,
connection=conn)
def email_devs(request):
form = DevMailerForm(request.POST or None)
preview = EmailPreviewTopic(topic='email-devs')
if preview.filter().count():
preview_csv = reverse('zadmin.email_preview_csv',
args=[preview.topic])
else:
preview_csv = None
if request.method == 'POST' and form.is_valid():
data = form.cleaned_data
if data['recipients'] == 'eula':
qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
amo.AUTHOR_ROLE_OWNER))
.exclude(addon__eula=None)
.exclude(user__email=None)
.distinct(['user__email']))
else:
raise NotImplementedError('If you want to support emailing other '
'types of developers, do it here!')
if data['preview_only']:
# Clear out the last batch of previewed emails.
preview.filter().delete()
total = 0
for emails in chunked(qs.values_list('user__email', flat=True), 100):
total += len(emails)
tasks.admin_email.delay(emails, data['subject'], data['message'],
preview_only=data['preview_only'],
preview_topic=preview.topic)
msg = 'Emails queued for delivery: %s' % total
if data['preview_only']:
msg = '%s (for preview only, emails not sent!)' % msg
messages.success(request, msg)
return redirect('zadmin.email_devs')
return jingo.render(request, 'zadmin/email-devs.html',
dict(form=form, preview_csv=preview_csv))
def handle(self, *args, **options):
pks = (ActivityLog.objects.review_queue().values_list('pk', flat=True)
.order_by('id'))
ts = [add_versionlog.subtask(args=[chunk])
for chunk in chunked(pks, 100)]
TaskSet(ts).apply_async()
def convert(directory, delete=False):
print 'Converting icons in %s' % directory
pks = []
k = 0
for path, names, filenames in walk_storage(directory):
for filename in filenames:
old = os.path.join(path, filename)
pre, ext = os.path.splitext(old)
if (pre[-3:] in size_suffixes or ext not in extensions):
continue
if not storage.size(old):
print 'Icon %s is empty, ignoring.' % old
continue
for size, size_suffix in zip(sizes, size_suffixes):
new = '%s%s%s' % (pre, size_suffix, '.png')
if os.path.exists(new):
continue
resize_image(old, new, (size, size), remove_src=False)
if ext != '.png':
pks.append(os.path.basename(pre))
if delete:
storage.delete(old)
k += 1
if not k % 1000:
print "... converted %s" % k
for chunk in chunked(pks, 100):
Addon.objects.filter(pk__in=chunk).update(icon_type='image/png')
def reindex_collections_task(index=None, aliased=True):
from . import tasks
ids = (Collection.objects.exclude(type=amo.COLLECTION_SYNCHRONIZED)
.values_list('id', flat=True))
taskset = [tasks.index_collections.si(chunk, index=index)
for chunk in chunked(sorted(list(ids)), 150)]
return group(taskset)
def deliver_hotness():
"""
Calculate hotness of all add-ons.
a = avg(users this week)
b = avg(users three weeks before this week)
hotness = (a-b) / b if a > 1000 and b > 1 else 0
"""
frozen = set(f.id for f in FrozenAddon.objects.all())
all_ids = list(
(Addon.objects.exclude(type=amo.ADDON_PERSONA).values_list('id',
flat=True)))
now = datetime.now()
one_week = now - timedelta(days=7)
four_weeks = now - timedelta(days=28)
for ids in chunked(all_ids, 300):
addons = Addon.objects.no_cache().filter(id__in=ids).no_transforms()
ids = [a.id for a in addons if a.id not in frozen]
qs = (UpdateCount.objects.filter(
addon__in=ids).values_list('addon').annotate(Avg('count')))
thisweek = dict(qs.filter(date__gte=one_week))
threeweek = dict(qs.filter(date__range=(four_weeks, one_week)))
for addon in addons:
this, three = thisweek.get(addon.id, 0), threeweek.get(addon.id, 0)
if this > 1000 and three > 1:
addon.update(hotness=(this - three) / float(three))
else:
addon.update(hotness=0)
# Let the database catch its breath.
time.sleep(10)
def handle(self, *args, **options):
if len(args) == 0:
raise CommandError('Please provide at least one add-on id to fix.')
addon_ids = [int(addon_id) for addon_id in args]
for chunk in chunked(addon_ids, 100):
fix_let_scope_bustage_in_addons.delay(chunk)
def reindex_collections():
from . import tasks
ids = (Collection.objects.exclude(type=amo.COLLECTION_SYNCHRONIZED)
.values_list('id', flat=True))
taskset = [tasks.index_collections.subtask(args=[chunk])
for chunk in chunked(sorted(list(ids)), 150)]
TaskSet(taskset).apply_async()
def activity_log_scrubber():
"""
Scans activity log for REMOVE_FROM_COLLECTION and ADD_TO_COLLECTION, looks
for collections in arguments and checks whether collection is listed.
"""
items = ActivityLog.objects.filter(
action__in=[amo.LOG.ADD_TO_COLLECTION.id,
amo.LOG.REMOVE_FROM_COLLECTION.id])
ids = []
count = 0
# ~127K
for item in items:
count += 1
for arg in item.arguments:
if isinstance(arg, Collection) and not arg.listed:
log.debug('Flagging %s.' % item)
log.debug('%d items seen.' % count)
ids.append(item.id)
log.info('Deleting %d items.' % len(ids))
for chunk in chunked(ids, 100):
_activity_log_scrubber.delay(chunk)
def update_app_trending():
"""Update trending for all apps."""
chunk_size = 300
all_ids = list(Webapp.objects.values_list('id', flat=True))
for ids in chunked(all_ids, chunk_size):
apps = Webapp.objects.filter(id__in=ids).no_transforms()
for app in apps:
# Calculate global trending, then per-region trending below.
value = _get_trending(app.id)
if value:
trending, created = app.trending.get_or_create(
region=0, defaults={'value': value})
if not created:
trending.update(value=value)
for region in mkt.regions.REGIONS_DICT.values():
value = _get_trending(app.id, region)
if value:
trending, created = app.trending.get_or_create(
region=region.id, defaults={'value': value})
if not created:
trending.update(value=value)
# Let the database catch its breath.
if len(all_ids) > chunk_size:
time.sleep(10)
def addon_total_contributions():
addons = Addon.objects.values_list('id', flat=True)
ts = [
tasks.addon_total_contributions.subtask(args=chunk)
for chunk in chunked(addons, 100)
]
TaskSet(ts).apply_async()
def update_user_ratings():
"""Update add-on author's ratings."""
cursor = connections[multidb.get_slave()].cursor()
# We build this query ahead of time because the cursor complains about data
# truncation if it does the parameters. Also, this query is surprisingly
# quick, <1sec for 6100 rows returned
q = """ SELECT
addons_users.user_id as user_id,
AVG(rating) as avg_rating
FROM reviews
INNER JOIN versions
INNER JOIN addons_users
INNER JOIN addons
ON reviews.version_id = versions.id
AND addons.id = versions.addon_id
AND addons_users.addon_id = addons.id
WHERE reviews.reply_to IS NULL
AND reviews.rating > 0
AND addons.status IN (%s)
GROUP BY addons_users.user_id
""" % (",".join(map(str, VALID_STATUSES)))
cursor.execute(q)
d = cursor.fetchall()
cursor.close()
ts = [update_user_ratings_task.subtask(args=[chunk])
for chunk in chunked(d, 1000)]
TaskSet(ts).apply_async()
def update_app_trending():
"""Update trending for all apps."""
chunk_size = 300
all_ids = list(Webapp.objects.values_list('id', flat=True))
for ids in chunked(all_ids, chunk_size):
apps = Webapp.objects.filter(id__in=ids).no_transforms()
for app in apps:
# Calculate global trending, then per-region trending below.
value = _get_trending(app.id)
if value:
trending, created = app.trending.get_or_create(
region=0, defaults={'value': value})
if not created:
trending.update(value=value)
for region in mkt.regions.REGIONS_DICT.values():
value = _get_trending(app.id, region)
if value:
trending, created = app.trending.get_or_create(
region=region.id, defaults={'value': value})
if not created:
trending.update(value=value)
# Let the database catch its breath.
if len(all_ids) > chunk_size:
time.sleep(10)
def convert(directory, delete=False):
print 'Converting icons in %s' % directory
pks = []
k = 0
for path, names, filenames in walk_storage(directory):
for filename in filenames:
old = os.path.join(path, filename)
pre, ext = os.path.splitext(old)
if (pre[-3:] in size_suffixes or ext not in extensions):
continue
if not storage.size(old):
print 'Icon %s is empty, ignoring.' % old
continue
for size, size_suffix in zip(sizes, size_suffixes):
new = '%s%s%s' % (pre, size_suffix, '.png')
if os.path.exists(new):
continue
resize_image(old, new, (size, size), remove_src=False)
if ext != '.png':
pks.append(os.path.basename(pre))
if delete:
storage.delete(old)
k += 1
if not k % 1000:
print "... converted %s" % k
for chunk in chunked(pks, 100):
Addon.objects.filter(pk__in=chunk).update(icon_type='image/png')
def handle(self, *args, **options):
task = tasks.get(options.get('task'))
if not task:
raise CommandError('Unknown task provided. Options are: %s'
% ', '.join(tasks.keys()))
pks = (Addon.objects.filter(*task['qs'])
.values_list('pk', flat=True)
.order_by('-last_updated'))
if 'pre' in task:
# This is run in process to ensure its run before the tasks.
pks = task['pre'](pks)
if pks:
kw = task.get('kwargs', {})
# All the remaining tasks go in one group.
grouping = []
for chunk in chunked(pks, 100):
grouping.append(
task['method'].subtask(args=[chunk], kwargs=kw))
# Add the post task on to the end.
post = None
if 'post' in task:
post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
ts = chord(grouping, post)
else:
ts = group(grouping)
ts.apply_async()
def mkt_gc(**kw):
"""Site-wide garbage collections."""
days_ago = lambda days: datetime.today() - timedelta(days=days)
log.debug('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))
for chunk in chunked(logs, 100):
chunk.sort()
log.debug('Deleting log entries: %s' % str(chunk))
amo.tasks.delete_logs.delay(chunk)
# Clear oauth nonce rows. These expire after 10 minutes but we're just
# clearing those that are more than 1 day old.
Nonce.objects.filter(created__lt=days_ago(1)).delete()
# Delete the dump apps over 30 days.
for app in os.listdir(settings.DUMPED_APPS_PATH):
app = os.path.join(settings.DUMPED_APPS_PATH, app)
if (os.stat(app).st_mtime < time.time() -
settings.DUMPED_APPS_DAYS_DELETE):
log.debug('Deleting old tarball: {0}'.format(app))
os.remove(app)
# Delete the dumped user installs over 30 days.
tarball_path = os.path.join(settings.DUMPED_USERS_PATH, 'tarballs')
for filename in os.listdir(tarball_path):
filepath = os.path.join(tarball_path, filename)
if (os.stat(filepath).st_mtime < time.time() -
settings.DUMPED_USERS_DAYS_DELETE):
log.debug('Deleting old tarball: {0}'.format(filepath))
os.remove(filepath)
def addon_reviews_ratings():
"""Update all add-on total_reviews and average/bayesian ratings."""
addons = Addon.objects.values_list('id', flat=True)
ts = [
tasks.cron_review_aggregate.subtask(args=chunk)
for chunk in chunked(addons, 100)
]
TaskSet(ts).apply_async()
def invalidate_users(**kw):
"""Invalidate all users to reflect latest Marketplace access whitelist."""
from amo.utils import chunked
from users.models import UserProfile
log.info('Invalidating users for access whitelist.')
d = UserProfile.objects.values_list('id', flat=True)
ts = [_invalidate_users.subtask(args=[chunk]) for chunk in chunked(d, 100)]
TaskSet(ts).apply_async()
def handle(self, *args, **options):
task = tasks.get(options.get('task'))
if not task:
raise ValueError('Unknown task: %s' % ', '.join(tasks.keys()))
pks = (Addon.objects.filter(*task['qs']).values_list(
'pk', flat=True).order_by('id'))
for chunk in chunked(pks, 100):
task['method'].delay(chunk)
def reviews_denorm():
"""Set is_latest and previous_count for all reviews."""
pairs = list(set(Review.objects.values_list('addon', 'user')))
ts = [
tasks.update_denorm.subtask(args=chunk)
for chunk in chunked(pairs, 50)
]
TaskSet(ts).apply_async()
def collection_meta():
from . import tasks
collections = Collection.objects.values_list('id', flat=True)
ts = [
tasks.cron_collection_meta.subtask(args=chunk)
for chunk in chunked(collections, 1000)
]
TaskSet(ts).apply_async()
def update_addons_collections_downloads():
"""Update addons+collections download totals."""
d = AddonCollectionCount.objects.values("addon", "collection").annotate(sum=Sum("count"))
with establish_connection() as conn:
for chunk in chunked(d, 600):
tasks.update_addons_collections_downloads.apply_async(args=[chunk], connection=conn)
def update_collections_total():
"""Update collections downloads totals."""
d = CollectionCount.objects.values("collection_id").annotate(sum=Sum("count"))
with establish_connection() as conn:
for chunk in chunked(d, 1000):
tasks.update_collections_total.apply_async(args=[chunk], connection=conn)
def reindex_users(index=None):
from . import tasks
ids = UserProfile.objects.values_list('id', flat=True)
taskset = [
tasks.index_users.subtask(args=[chunk], kwargs=dict(index=index))
for chunk in chunked(sorted(list(ids)), 150)
]
TaskSet(taskset).apply_async()
def run():
"""Delete duplicate image assets."""
for chunk in chunked(Webapp.objects.all(), 50):
for app in chunk:
for slug in SIZE_SLUGS:
assets = ImageAsset.objects.filter(addon=app, slug=slug)
for asset in assets[1:]:
asset.delete()
def update_addons_current_version():
"""Update the current_version field of the addons."""
d = Addon.objects.valid().exclude(type=amo.ADDON_PERSONA).values_list('id')
with establish_connection() as conn:
for chunk in chunked(d, 1000):
_update_addons_current_version.apply_async(args=[chunk],
connection=conn)
def run():
"""Delete duplicate image assets."""
for chunk in chunked(Webapp.objects.all(), 50):
for app in chunk:
for slug in SIZE_SLUGS:
assets = ImageAsset.objects.filter(addon=app, slug=slug)
for asset in assets[1:]:
asset.delete()
def invalidate_users(**kw):
"""Invalidate all users to reflect latest Marketplace access whitelist."""
from amo.utils import chunked
from users.models import UserProfile
log.info('Invalidating users for access whitelist.')
d = UserProfile.objects.values_list('id', flat=True)
ts = [_invalidate_users.subtask(args=[chunk]) for chunk in chunked(d, 100)]
TaskSet(ts).apply_async()
def handle(self, *args, **options):
activity_ids = AppLog.objects.values_list('activity_log', flat=True)
logs = (ActivityLog.objects.filter(
pk__in=list(activity_ids), action__in=amo.LOG_REVIEW_QUEUE)
.order_by('created'))
for log_chunk in chunked(logs, 100):
_migrate_activity_log.delay(log_chunk)
def flip_webapp_status(from_, to):
qs = Webapp.objects.filter(status=from_)
# Grab the ids so we can get them reindexed.
ids = list(qs.values_list('id', flat=True))
qs.update(status=to)
ts = [index_addons.subtask(args=[chunk]) for chunk in chunked(ids, 150)]
# Delay these tasks to avoid slave lag.
TaskSet(ts).apply_async(countdown=30)
def update_all_appsupport():
from .tasks import update_appsupport
ids = sorted(set(AppSupport.objects.values_list('addon', flat=True)))
task_log.info('Updating appsupport for %s addons.' % len(ids))
for idx, chunk in enumerate(chunked(ids, 100)):
if idx % 10 == 0:
task_log.info('[%s/%s] Updating appsupport.'
% (idx * 100, len(ids)))
update_appsupport(chunk)
def update_collections_total():
"""Update collections downloads totals."""
d = (CollectionCount.objects.values('collection_id')
.annotate(sum=Sum('count')))
ts = [tasks.update_collections_total.subtask(args=[chunk])
for chunk in chunked(d, 50)]
TaskSet(ts).apply_async()
def reindex_collections(index=None):
from . import tasks
ids = (Collection.objects.exclude(
type=amo.COLLECTION_SYNCHRONIZED).values_list('id', flat=True))
taskset = [
tasks.index_collections.subtask(args=[chunk], kwargs=dict(index=index))
for chunk in chunked(sorted(list(ids)), 150)
]
TaskSet(taskset).apply_async()
def featured_grid(context, addons, src=None, dl_src=None, pagesize=3, cols=3):
if not src:
src = context.get('src')
# dl_src is an optional src paramater just for the download links
if not dl_src:
dl_src = src
pages = chunked(addons, pagesize)
columns = '' if cols != 3 else 'three-col'
return new_context(**locals())
def update_addons_collections_downloads():
"""Update addons+collections download totals."""
d = (AddonCollectionCount.objects.values('addon', 'collection')
.annotate(sum=Sum('count')))
ts = [tasks.update_addons_collections_downloads.subtask(args=[chunk])
for chunk in chunked(d, 600)]
TaskSet(ts).apply_async()
def update_collections_total():
"""Update collections downloads totals."""
d = (CollectionCount.objects.values('collection_id')
.annotate(sum=Sum('count')))
ts = [tasks.update_collections_total.subtask(args=[chunk])
for chunk in chunked(d, 1000)]
TaskSet(ts).apply_async()
def update_all_appsupport():
from .tasks import update_appsupport
ids = sorted(set(AppSupport.objects.values_list('addon', flat=True)))
task_log.info('Updating appsupport for %s addons.' % len(ids))
for idx, chunk in enumerate(chunked(ids, 100)):
if idx % 10 == 0:
task_log.info('[%s/%s] Updating appsupport.' %
(idx * 100, len(ids)))
update_appsupport(chunk)
