Esempio n. 1
0
def _task(**kw):
    # Remove any dupes. `UNIQUE` constraint introduced in migration 504.
    dupes = (ADT.objects.values_list(
        'addon', 'device_type').annotate(c=Count('id')).filter(c__gt=1))
    for addon, device_type, total in dupes:
        devices = ADT.objects.filter(addon_id=addon, device_type=device_type)
        for d in devices[:total - 1]:
            d.delete()

    # Remove stale device types.
    devices = ADT.objects.all()
    for chunk in chunked(devices, 50):
        for device in chunk:
            try:
                device.addon
            except ObjectDoesNotExist:
                device.delete()

    # `DEVICE_MOBILE` -> `DEVICE_MOBILE` and `DEVICE_GAIA`.
    devices = ADT.objects.filter(device_type=amo.DEVICE_MOBILE.id)

    for chunk in chunked(devices, 50):
        for device in chunk:
            if amo.DEVICE_GAIA in device.addon.device_types:
                continue
            device.id = None
            device.device_type = amo.DEVICE_GAIA.id
            device.save()
            device.addon.save()
Esempio n. 2
0
def _task(**kw):
    # Remove any dupes. `UNIQUE` constraint introduced in migration 504.
    dupes = (ADT.objects.values_list('addon', 'device_type')
                        .annotate(c=Count('id')).filter(c__gt=1))
    for addon, device_type, total in dupes:
        devices = ADT.objects.filter(addon_id=addon, device_type=device_type)
        for d in devices[:total - 1]:
            d.delete()

    # Remove stale device types.
    devices = ADT.objects.all()
    for chunk in chunked(devices, 50):
        for device in chunk:
            try:
                device.addon
            except ObjectDoesNotExist:
                device.delete()

    # `DEVICE_MOBILE` -> `DEVICE_MOBILE` and `DEVICE_GAIA`.
    devices = ADT.objects.filter(device_type=amo.DEVICE_MOBILE.id)

    for chunk in chunked(devices, 50):
        for device in chunk:
            if amo.DEVICE_GAIA in device.addon.device_types:
                continue
            device.id = None
            device.device_type = amo.DEVICE_GAIA.id
            device.save()
            device.addon.save()
Esempio n. 3
0
def notify(request, job):
    job = get_object_or_404(ValidationJob, pk=job)
    notify_form = NotifyForm(request.POST, text='validation')

    if not notify_form.is_valid():
        messages.error(request, notify_form)
    else:
        for chunk in chunked(updated_versions(job), 100):
            tasks.update_maxversions.delay(chunk, job.pk,
                                           notify_form.cleaned_data)

        updated_authors = completed_version_authors(job)
        for chunk in chunked(updated_authors, 100):
            # There are times when you want to punch django's ORM in
            # the face. This may be one of those times.
            # TODO: Offload most of this work to the task?
            users_addons = list(
                UserProfile.objects.filter(
                    pk__in=chunk).filter(**_completed_versions(
                        job, 'addons__versions')).values_list(
                            'pk', 'addons__pk').distinct())

            users = list(
                UserProfile.objects.filter(pk__in=set(
                    u for u, a in users_addons)))

            # Annotate fails in tests when using cached results
            addons = (Addon.objects.no_cache().filter(
                **{
                    'pk__in': set(a for u, a in users_addons),
                    'versions__files__'
                    'validation_results__validation_job': job
                }).annotate(
                    errors=Sum('versions__files__validation_results__errors')))
            addons = dict((a.id, a) for a in addons)

            users_addons = dict(
                (u, [addons[a] for u, a in row])
                for u, row in sorted_groupby(users_addons, lambda k: k[0]))

            for u in users:
                addons = users_addons[u.pk]

                u.passing_addons = [a for a in addons if a.errors == 0]
                u.failing_addons = [a for a in addons if a.errors > 0]

            tasks.notify_compatibility.delay(users, job,
                                             notify_form.cleaned_data)

    return redirect(reverse('zadmin.validation'))
Esempio n. 4
0
def notify(request, job):
    job = get_object_or_404(ValidationJob, pk=job)
    notify_form = NotifyForm(request.POST, text='validation')

    if not notify_form.is_valid():
        messages.error(request, notify_form)
    else:
        for chunk in chunked(updated_versions(job), 100):
            tasks.update_maxversions.delay(chunk, job.pk,
                                           notify_form.cleaned_data)

        updated_authors = completed_version_authors(job)
        for chunk in chunked(updated_authors, 100):
            # There are times when you want to punch django's ORM in
            # the face. This may be one of those times.
            # TODO: Offload most of this work to the task?
            users_addons = list(
                UserProfile.objects.filter(pk__in=chunk)
                           .filter(**_completed_versions(job,
                                                         'addons__versions'))
                           .values_list('pk', 'addons__pk').distinct())

            users = list(UserProfile.objects.filter(
                             pk__in=set(u for u, a in users_addons)))

            # Annotate fails in tests when using cached results
            addons = (Addon.objects.no_cache()
                           .filter(**{
                               'pk__in': set(a for u, a in users_addons),
                               'versions__files__'
                                    'validation_results__validation_job': job
                           })
                           .annotate(errors=Sum(
                               'versions__files__validation_results__errors')))
            addons = dict((a.id, a) for a in addons)

            users_addons = dict((u, [addons[a] for u, a in row])
                                for u, row in sorted_groupby(users_addons,
                                                             lambda k: k[0]))

            for u in users:
                addons = users_addons[u.pk]

                u.passing_addons = [a for a in addons if a.errors == 0]
                u.failing_addons = [a for a in addons if a.errors > 0]

            tasks.notify_compatibility.delay(users, job,
                                             notify_form.cleaned_data)

    return redirect(reverse('zadmin.validation'))
Esempio n. 5
0
def reindex_users(index=None):
    from . import tasks
    ids = UserProfile.objects.values_list('id', flat=True)
    taskset = [tasks.index_users.subtask(args=[chunk],
                                         kwargs=dict(index=index))
               for chunk in chunked(sorted(list(ids)), 150)]
    TaskSet(taskset).apply_async()
Esempio n. 6
0
def deliver_hotness():
    """
    Calculate hotness of all add-ons.

    a = avg(users this week)
    b = avg(users three weeks before this week)
    hotness = (a-b) / b if a > 1000 and b > 1 else 0
    """
    frozen = set(f.id for f in FrozenAddon.objects.all())
    all_ids = list((Addon.objects.exclude(type=amo.ADDON_PERSONA)
                   .values_list('id', flat=True)))
    now = datetime.now()
    one_week = now - timedelta(days=7)
    four_weeks = now - timedelta(days=28)
    for ids in chunked(all_ids, 300):
        addons = Addon.objects.no_cache().filter(id__in=ids).no_transforms()
        ids = [a.id for a in addons if a.id not in frozen]
        qs = (UpdateCount.objects.filter(addon__in=ids)
              .values_list('addon').annotate(Avg('count')))
        thisweek = dict(qs.filter(date__gte=one_week))
        threeweek = dict(qs.filter(date__range=(four_weeks, one_week)))
        for addon in addons:
            this, three = thisweek.get(addon.id, 0), threeweek.get(addon.id, 0)
            if this > 1000 and three > 1:
                addon.update(hotness=(this - three) / float(three))
            else:
                addon.update(hotness=0)
        # Let the database catch its breath.
        time.sleep(10)
Esempio n. 7
0
 def handle(self, *args, **kw):
     qs = Version.objects.filter(version_int=None)
     print 'Found %s versions that need updating' % qs.count()
     with establish_connection() as conn:
         for pks in chunked(list(qs.values_list('pk', flat=True)), 1000):
             add_version_int.delay(pks)
     print '... added to celery.'
Esempio n. 8
0
def mkt_gc(**kw):
    """Site-wide garbage collections."""
    days_ago = lambda days: datetime.today() - timedelta(days=days)

    log.debug('Collecting data to delete')
    logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude(
        action__in=amo.LOG_KEEP).values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        chunk.sort()
        log.debug('Deleting log entries: %s' % str(chunk))
        amo.tasks.delete_logs.delay(chunk)

    # Delete the dump apps over 30 days.
    for app in os.listdir(settings.DUMPED_APPS_PATH):
        app = os.path.join(settings.DUMPED_APPS_PATH, app)
        if (os.stat(app).st_mtime <
                time.time() - settings.DUMPED_APPS_DAYS_DELETE):
            log.debug('Deleting old tarball: {0}'.format(app))
            os.remove(app)

    # Delete the dumped user installs over 30 days.
    tarball_path = os.path.join(settings.DUMPED_USERS_PATH, 'tarballs')
    for filename in os.listdir(tarball_path):
        filepath = os.path.join(tarball_path, filename)
        if (os.stat(filepath).st_mtime <
                time.time() - settings.DUMPED_USERS_DAYS_DELETE):
            log.debug('Deleting old tarball: {0}'.format(filepath))
            os.remove(filepath)
Esempio n. 9
0
    def handle(self, *args, **options):
        # Avoiding loops
        from amo.utils import chunked, slugify
        from users.models import UserProfile
        from users.tasks import _delete_users

        if not args:
            print "Usage: manage.py delete_users <file>"
            sys.exit(1)


        if not os.path.exists(args[0]):
            print "File not found: %s" % args[0]
            sys.exit(1)

        f = open(args[0], 'r')

        data = True

        print "Reading %s" % args[0]

        while data:
            data = f.readlines(100000)  # 100000 is about 13500 user ids
            data = [x.strip() for x in data]  # has newlines on it

            print "Sending %s users to celery" % len(data)

            with establish_connection() as conn:
                for chunk in chunked(data, 100):
                    _delete_users.apply_async(args=[chunk], connection=conn)

        f.close()

        print "All done."
    def handle(self, *args, **kwargs):

        ids = []

        ids.extend(
            FeedCollectionMembership.objects.values_list('group', flat=True))
        ids.extend(
            FeedCollection.objects.values_list('name', flat=True))
        ids.extend(
            FeedCollection.objects.values_list('description', flat=True))
        ids.extend(
            FeedShelfMembership.objects.values_list('group', flat=True))
        ids.extend(
            FeedShelf.objects.values_list('description', flat=True))
        ids.extend(
            FeedShelf.objects.values_list('name', flat=True))
        ids.extend(
            FeedApp.objects.values_list('description', flat=True))
        ids.extend(
            FeedApp.objects.values_list('pullquote_text', flat=True))
        ids.extend(
            Version.objects.values_list('releasenotes', flat=True))
        ids.extend(
            Webapp.objects.values_list('description', flat=True))
        ids.extend(
            Webapp.objects.values_list('privacy_policy', flat=True))
        ids.extend(
            Geodata.objects.values_list('banner_message', flat=True))

        # Filter out any None's.
        ids = filter(None, ids)

        for chunk in chunked(ids, 100):
            update_translations.delay(chunk)
Esempio n. 11
0
def email_devs(request):
    form = DevMailerForm(request.POST or None)
    preview = EmailPreviewTopic(topic='email-devs')
    if preview.filter().count():
        preview_csv = reverse('zadmin.email_preview_csv',
                              args=[preview.topic])
    else:
        preview_csv = None
    if request.method == 'POST' and form.is_valid():
        data = form.cleaned_data
        qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
                                                 amo.AUTHOR_ROLE_OWNER))
                               .exclude(user__email=None))

        if data['recipients'] in ('payments', 'desktop_apps'):
            qs = qs.exclude(addon__status=amo.STATUS_DELETED)
        else:
            qs = qs.filter(addon__status__in=amo.LISTED_STATUSES)

        if data['recipients'] == 'eula':
            qs = qs.exclude(addon__eula=None)
        elif data['recipients'] in ('payments',
                                    'payments_region_enabled',
                                    'payments_region_disabled'):
            qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
            qs = qs.exclude(addon__premium_type__in=(amo.ADDON_FREE,
                                                     amo.ADDON_OTHER_INAPP))
            if data['recipients'] == 'payments_region_enabled':
                qs = qs.filter(addon__enable_new_regions=True)
            elif data['recipients'] == 'payments_region_disabled':
                qs = qs.filter(addon__enable_new_regions=False)
        elif data['recipients'] == 'apps':
            qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
        elif data['recipients'] == 'desktop_apps':
            qs = (qs.filter(addon__type=amo.ADDON_WEBAPP,
                addon__addondevicetype__device_type=amo.DEVICE_DESKTOP.id))
        elif data['recipients'] == 'sdk':
            qs = qs.exclude(addon__versions__files__jetpack_version=None)
        elif data['recipients'] == 'all_extensions':
            qs = qs.filter(addon__type=amo.ADDON_EXTENSION)
        else:
            raise NotImplementedError('If you want to support emailing other '
                                      'types of developers, do it here!')
        if data['preview_only']:
            # Clear out the last batch of previewed emails.
            preview.filter().delete()
        total = 0
        for emails in chunked(set(qs.values_list('user__email', flat=True)),
                              100):
            total += len(emails)
            tasks.admin_email.delay(emails, data['subject'], data['message'],
                                    preview_only=data['preview_only'],
                                    preview_topic=preview.topic)
        msg = 'Emails queued for delivery: %s' % total
        if data['preview_only']:
            msg = '%s (for preview only, emails not sent!)' % msg
        messages.success(request, msg)
        return redirect('zadmin.email_devs')
    return jingo.render(request, 'zadmin/email-devs.html',
                        dict(form=form, preview_csv=preview_csv))
Esempio n. 12
0
def run():
    for chunk in chunked(Webapp.objects.all(), 50):
        for app in chunk:
            try:
                generate_image_assets.delay(app)
            except Exception:
                pass
    def handle(self, *args, **options):
        ids = (CommunicationNote.objects
                                .filter(note_type=comm.REVIEWER_COMMENT)
                                .values_list('id', flat=True))

        for log_chunk in chunked(ids, 100):
            _fix_developer_version_notes.delay(ids)
Esempio n. 14
0
def update_perf():
    cursor = connections[multidb.get_slave()].cursor()
    # The baseline is where addon_id is null.
    cursor.execute(
        "SELECT AVG(average) FROM perf_results WHERE addon_id IS NULL")
    baseline = cursor.fetchone()[0]

    # The perf_results table is a mess right now, so pull out one row
    # for each addon by finding the MAX(created) and then the AVG(average)
    # since there are many rows with the same (addon, created).
    # This scheme completely ignores app, os, and test.
    cursor.execute("""
        SELECT J.addon_id, AVG(average) av FROM perf_results P INNER JOIN
            (SELECT addon_id, MAX(created) c FROM perf_results
             GROUP BY addon_id) J
        ON ((P.addon_id=J.addon_id) AND P.created=J.c)
        WHERE test='ts'
        GROUP BY P.addon_id
        HAVING av > %s""", (baseline,))
    # A bunch of (addon, perf_average) pairs.
    perf = cursor.fetchall()
    with establish_connection() as conn:
        for chunk in chunked(perf, 25):
            tasks.update_perf.apply_async(args=[baseline, chunk],
                                          connection=conn)
    cursor.close()
Esempio n. 15
0
def email_devs(request):
    form = DevMailerForm(request.POST or None)
    preview = EmailPreviewTopic(topic='email-devs')
    if preview.filter().count():
        preview_csv = reverse('zadmin.email_preview_csv',
                              args=[preview.topic])
    else:
        preview_csv = None
    if request.method == 'POST' and form.is_valid():
        data = form.cleaned_data
        qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
                                                 amo.AUTHOR_ROLE_OWNER))
                               .exclude(user__email=None))

        if data['recipients'] in ('payments', 'desktop_apps'):
            qs = qs.exclude(addon__status=amo.STATUS_DELETED)
        else:
            qs = qs.filter(addon__status__in=amo.LISTED_STATUSES)

        if data['recipients'] == 'eula':
            qs = qs.exclude(addon__eula=None)
        elif data['recipients'] in ('payments',
                                    'payments_region_enabled',
                                    'payments_region_disabled'):
            qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
            qs = qs.exclude(addon__premium_type__in=(amo.ADDON_FREE,
                                                     amo.ADDON_OTHER_INAPP))
            if data['recipients'] == 'payments_region_enabled':
                qs = qs.filter(addon__enable_new_regions=True)
            elif data['recipients'] == 'payments_region_disabled':
                qs = qs.filter(addon__enable_new_regions=False)
        elif data['recipients'] == 'apps':
            qs = qs.filter(addon__type=amo.ADDON_WEBAPP)
        elif data['recipients'] == 'desktop_apps':
            qs = (qs.filter(addon__type=amo.ADDON_WEBAPP,
                addon__addondevicetype__device_type=amo.DEVICE_DESKTOP.id))
        elif data['recipients'] == 'sdk':
            qs = qs.exclude(addon__versions__files__jetpack_version=None)
        elif data['recipients'] == 'all_extensions':
            qs = qs.filter(addon__type=amo.ADDON_EXTENSION)
        else:
            raise NotImplementedError('If you want to support emailing other '
                                      'types of developers, do it here!')
        if data['preview_only']:
            # Clear out the last batch of previewed emails.
            preview.filter().delete()
        total = 0
        for emails in chunked(set(qs.values_list('user__email', flat=True)),
                              100):
            total += len(emails)
            tasks.admin_email.delay(emails, data['subject'], data['message'],
                                    preview_only=data['preview_only'],
                                    preview_topic=preview.topic)
        msg = 'Emails queued for delivery: %s' % total
        if data['preview_only']:
            msg = '%s (for preview only, emails not sent!)' % msg
        messages.success(request, msg)
        return redirect('zadmin.email_devs')
    return jingo.render(request, 'zadmin/email-devs.html',
                        dict(form=form, preview_csv=preview_csv))
Esempio n. 16
0
def update_user_ratings():
    """Update add-on author's ratings."""

    cursor = connections[multidb.get_slave()].cursor()
    # We build this query ahead of time because the cursor complains about data
    # truncation if it does the parameters.  Also, this query is surprisingly
    # quick, <1sec for 6100 rows returned
    q = """   SELECT
                addons_users.user_id as user_id,
                AVG(rating) as avg_rating
              FROM reviews
                INNER JOIN versions
                INNER JOIN addons_users
                INNER JOIN addons
              ON reviews.version_id = versions.id
                AND addons.id = versions.addon_id
                AND addons_users.addon_id = addons.id
              WHERE reviews.reply_to IS NULL
                AND reviews.rating > 0
                AND addons.status IN (%s)
              GROUP BY addons_users.user_id
              """ % (",".join(map(str, VALID_STATUSES)))

    cursor.execute(q)
    d = cursor.fetchall()
    cursor.close()

    with establish_connection() as conn:
        for chunk in chunked(d, 1000):
            _update_user_ratings.apply_async(args=[chunk], connection=conn)
Esempio n. 17
0
    def handle(self, *args, **options):
        task = tasks.get(options.get('task'))
        if not task:
            raise CommandError('Unknown task provided. Options are: %s' %
                               ', '.join(tasks.keys()))
        pks = (Webapp.objects.filter(*task['qs']).values_list(
            'pk', flat=True).order_by('-last_updated'))
        if 'pre' in task:
            # This is run in process to ensure its run before the tasks.
            pks = task['pre'](pks)
        if pks:
            kw = task.get('kwargs', {})
            # All the remaining tasks go in one group.
            grouping = []
            for chunk in chunked(pks, 100):
                grouping.append(task['method'].subtask(args=[chunk],
                                                       kwargs=kw))

            # Add the post task on to the end.
            post = None
            if 'post' in task:
                post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
                ts = chord(grouping, post)
            else:
                ts = group(grouping)
            ts.apply_async()
Esempio n. 18
0
def addon_reviews_ratings():
    """Update all add-on total_reviews and average/bayesian ratings."""
    addons = Addon.objects.values_list('id', flat=True)
    with establish_connection() as conn:
        for chunk in chunked(addons, 100):
            tasks.cron_review_aggregate.apply_async(args=chunk,
                                                    connection=conn)
Esempio n. 19
0
def email_devs(request):
    form = DevMailerForm(request.POST or None)
    preview = EmailPreviewTopic(topic='email-devs')
    if preview.filter().count():
        preview_csv = reverse('zadmin.email_preview_csv',
                              args=[preview.topic])
    else:
        preview_csv = None
    if request.method == 'POST' and form.is_valid():
        data = form.cleaned_data
        if data['recipients'] == 'eula':
            qs = (AddonUser.objects.filter(role__in=(amo.AUTHOR_ROLE_DEV,
                                                     amo.AUTHOR_ROLE_OWNER))
                                   .exclude(addon__eula=None)
                                   .exclude(user__email=None)
                                   .distinct(['user__email']))
        else:
            raise NotImplementedError('If you want to support emailing other '
                                      'types of developers, do it here!')
        if data['preview_only']:
            # Clear out the last batch of previewed emails.
            preview.filter().delete()
        total = 0
        for emails in chunked(qs.values_list('user__email', flat=True), 100):
            total += len(emails)
            tasks.admin_email.delay(emails, data['subject'], data['message'],
                                    preview_only=data['preview_only'],
                                    preview_topic=preview.topic)
        msg = 'Emails queued for delivery: %s' % total
        if data['preview_only']:
            msg = '%s (for preview only, emails not sent!)' % msg
        messages.success(request, msg)
        return redirect('zadmin.email_devs')
    return jingo.render(request, 'zadmin/email-devs.html',
                        dict(form=form, preview_csv=preview_csv))
Esempio n. 20
0
    def handle(self, *args, **options):
        pks = (ActivityLog.objects.review_queue().values_list('pk', flat=True)
                                  .order_by('id'))

        ts = [add_versionlog.subtask(args=[chunk])
              for chunk in chunked(pks, 100)]
        TaskSet(ts).apply_async()
Esempio n. 21
0
def convert(directory, delete=False):
    print 'Converting icons in %s' % directory

    pks = []
    k = 0
    for path, names, filenames in walk_storage(directory):
        for filename in filenames:
            old = os.path.join(path, filename)
            pre, ext = os.path.splitext(old)
            if (pre[-3:] in size_suffixes or ext not in extensions):
                continue

            if not storage.size(old):
                print 'Icon %s is empty, ignoring.' % old
                continue

            for size, size_suffix in zip(sizes, size_suffixes):
                new = '%s%s%s' % (pre, size_suffix, '.png')
                if os.path.exists(new):
                    continue
                resize_image(old, new, (size, size), remove_src=False)

            if ext != '.png':
                pks.append(os.path.basename(pre))

            if delete:
                storage.delete(old)

            k += 1
            if not k % 1000:
                print "... converted %s" % k

    for chunk in chunked(pks, 100):
        Addon.objects.filter(pk__in=chunk).update(icon_type='image/png')
Esempio n. 22
0
def reindex_collections_task(index=None, aliased=True):
    from . import tasks
    ids = (Collection.objects.exclude(type=amo.COLLECTION_SYNCHRONIZED)
           .values_list('id', flat=True))
    taskset = [tasks.index_collections.si(chunk, index=index)
               for chunk in chunked(sorted(list(ids)), 150)]
    return group(taskset)
Esempio n. 23
0
def deliver_hotness():
    """
    Calculate hotness of all add-ons.

    a = avg(users this week)
    b = avg(users three weeks before this week)
    hotness = (a-b) / b if a > 1000 and b > 1 else 0
    """
    frozen = set(f.id for f in FrozenAddon.objects.all())
    all_ids = list(
        (Addon.objects.exclude(type=amo.ADDON_PERSONA).values_list('id',
                                                                   flat=True)))
    now = datetime.now()
    one_week = now - timedelta(days=7)
    four_weeks = now - timedelta(days=28)
    for ids in chunked(all_ids, 300):
        addons = Addon.objects.no_cache().filter(id__in=ids).no_transforms()
        ids = [a.id for a in addons if a.id not in frozen]
        qs = (UpdateCount.objects.filter(
            addon__in=ids).values_list('addon').annotate(Avg('count')))
        thisweek = dict(qs.filter(date__gte=one_week))
        threeweek = dict(qs.filter(date__range=(four_weeks, one_week)))
        for addon in addons:
            this, three = thisweek.get(addon.id, 0), threeweek.get(addon.id, 0)
            if this > 1000 and three > 1:
                addon.update(hotness=(this - three) / float(three))
            else:
                addon.update(hotness=0)
        # Let the database catch its breath.
        time.sleep(10)
    def handle(self, *args, **options):
        if len(args) == 0:
            raise CommandError('Please provide at least one add-on id to fix.')

        addon_ids = [int(addon_id) for addon_id in args]
        for chunk in chunked(addon_ids, 100):
            fix_let_scope_bustage_in_addons.delay(chunk)
Esempio n. 25
0
def reindex_collections():
    from . import tasks
    ids = (Collection.objects.exclude(type=amo.COLLECTION_SYNCHRONIZED)
           .values_list('id', flat=True))
    taskset = [tasks.index_collections.subtask(args=[chunk])
               for chunk in chunked(sorted(list(ids)), 150)]
    TaskSet(taskset).apply_async()
Esempio n. 26
0
def activity_log_scrubber():
    """
    Scans activity log for REMOVE_FROM_COLLECTION and ADD_TO_COLLECTION, looks
    for collections in arguments and checks whether collection is listed.
    """

    items = ActivityLog.objects.filter(
            action__in=[amo.LOG.ADD_TO_COLLECTION.id,
                        amo.LOG.REMOVE_FROM_COLLECTION.id])

    ids = []
    count = 0
    # ~127K
    for item in items:
        count += 1
        for arg in item.arguments:
            if isinstance(arg, Collection) and not arg.listed:
                log.debug('Flagging %s.' % item)
                log.debug('%d items seen.' % count)
                ids.append(item.id)

    log.info('Deleting %d items.' % len(ids))

    for chunk in chunked(ids, 100):
        _activity_log_scrubber.delay(chunk)
Esempio n. 27
0
def update_app_trending():
    """Update trending for all apps."""
    chunk_size = 300
    all_ids = list(Webapp.objects.values_list('id', flat=True))

    for ids in chunked(all_ids, chunk_size):
        apps = Webapp.objects.filter(id__in=ids).no_transforms()
        for app in apps:
            # Calculate global trending, then per-region trending below.
            value = _get_trending(app.id)
            if value:
                trending, created = app.trending.get_or_create(
                    region=0, defaults={'value': value})
                if not created:
                    trending.update(value=value)

            for region in mkt.regions.REGIONS_DICT.values():
                value = _get_trending(app.id, region)
                if value:
                    trending, created = app.trending.get_or_create(
                        region=region.id, defaults={'value': value})
                    if not created:
                        trending.update(value=value)

        # Let the database catch its breath.
        if len(all_ids) > chunk_size:
            time.sleep(10)
Esempio n. 28
0
def addon_total_contributions():
    addons = Addon.objects.values_list('id', flat=True)
    ts = [
        tasks.addon_total_contributions.subtask(args=chunk)
        for chunk in chunked(addons, 100)
    ]
    TaskSet(ts).apply_async()
Esempio n. 29
0
def update_user_ratings():
    """Update add-on author's ratings."""

    cursor = connections[multidb.get_slave()].cursor()
    # We build this query ahead of time because the cursor complains about data
    # truncation if it does the parameters.  Also, this query is surprisingly
    # quick, <1sec for 6100 rows returned
    q = """   SELECT
                addons_users.user_id as user_id,
                AVG(rating) as avg_rating
              FROM reviews
                INNER JOIN versions
                INNER JOIN addons_users
                INNER JOIN addons
              ON reviews.version_id = versions.id
                AND addons.id = versions.addon_id
                AND addons_users.addon_id = addons.id
              WHERE reviews.reply_to IS NULL
                AND reviews.rating > 0
                AND addons.status IN (%s)
              GROUP BY addons_users.user_id
              """ % (",".join(map(str, VALID_STATUSES)))

    cursor.execute(q)
    d = cursor.fetchall()
    cursor.close()

    ts = [update_user_ratings_task.subtask(args=[chunk])
          for chunk in chunked(d, 1000)]
    TaskSet(ts).apply_async()
Esempio n. 30
0
def update_app_trending():
    """Update trending for all apps."""
    chunk_size = 300
    all_ids = list(Webapp.objects.values_list('id', flat=True))

    for ids in chunked(all_ids, chunk_size):
        apps = Webapp.objects.filter(id__in=ids).no_transforms()
        for app in apps:
            # Calculate global trending, then per-region trending below.
            value = _get_trending(app.id)
            if value:
                trending, created = app.trending.get_or_create(
                    region=0, defaults={'value': value})
                if not created:
                    trending.update(value=value)

            for region in mkt.regions.REGIONS_DICT.values():
                value = _get_trending(app.id, region)
                if value:
                    trending, created = app.trending.get_or_create(
                        region=region.id, defaults={'value': value})
                    if not created:
                        trending.update(value=value)

        # Let the database catch its breath.
        if len(all_ids) > chunk_size:
            time.sleep(10)
Esempio n. 31
0
def convert(directory, delete=False):
    print 'Converting icons in %s' % directory

    pks = []
    k = 0
    for path, names, filenames in walk_storage(directory):
        for filename in filenames:
            old = os.path.join(path, filename)
            pre, ext = os.path.splitext(old)
            if (pre[-3:] in size_suffixes or ext not in extensions):
                continue

            if not storage.size(old):
                print 'Icon %s is empty, ignoring.' % old
                continue

            for size, size_suffix in zip(sizes, size_suffixes):
                new = '%s%s%s' % (pre, size_suffix, '.png')
                if os.path.exists(new):
                    continue
                resize_image(old, new, (size, size), remove_src=False)

            if ext != '.png':
                pks.append(os.path.basename(pre))

            if delete:
                storage.delete(old)

            k += 1
            if not k % 1000:
                print "... converted %s" % k

    for chunk in chunked(pks, 100):
        Addon.objects.filter(pk__in=chunk).update(icon_type='image/png')
Esempio n. 32
0
    def handle(self, *args, **options):
        task = tasks.get(options.get('task'))
        if not task:
            raise CommandError('Unknown task provided. Options are: %s'
                               % ', '.join(tasks.keys()))
        pks = (Addon.objects.filter(*task['qs'])
                            .values_list('pk', flat=True)
                            .order_by('-last_updated'))
        if 'pre' in task:
            # This is run in process to ensure its run before the tasks.
            pks = task['pre'](pks)
        if pks:
            kw = task.get('kwargs', {})
            # All the remaining tasks go in one group.
            grouping = []
            for chunk in chunked(pks, 100):
                grouping.append(
                    task['method'].subtask(args=[chunk], kwargs=kw))

            # Add the post task on to the end.
            post = None
            if 'post' in task:
                post = task['post'].subtask(args=[], kwargs=kw, immutable=True)
                ts = chord(grouping, post)
            else:
                ts = group(grouping)
            ts.apply_async()
Esempio n. 33
0
def mkt_gc(**kw):
    """Site-wide garbage collections."""
    days_ago = lambda days: datetime.today() - timedelta(days=days)

    log.debug('Collecting data to delete')
    logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
            .exclude(action__in=amo.LOG_KEEP).values_list('id', flat=True))

    for chunk in chunked(logs, 100):
        chunk.sort()
        log.debug('Deleting log entries: %s' % str(chunk))
        amo.tasks.delete_logs.delay(chunk)

    # Clear oauth nonce rows. These expire after 10 minutes but we're just
    # clearing those that are more than 1 day old.
    Nonce.objects.filter(created__lt=days_ago(1)).delete()

    # Delete the dump apps over 30 days.
    for app in os.listdir(settings.DUMPED_APPS_PATH):
        app = os.path.join(settings.DUMPED_APPS_PATH, app)
        if (os.stat(app).st_mtime < time.time() -
            settings.DUMPED_APPS_DAYS_DELETE):
            log.debug('Deleting old tarball: {0}'.format(app))
            os.remove(app)

    # Delete the dumped user installs over 30 days.
    tarball_path = os.path.join(settings.DUMPED_USERS_PATH, 'tarballs')
    for filename in os.listdir(tarball_path):
        filepath = os.path.join(tarball_path, filename)
        if (os.stat(filepath).st_mtime < time.time() -
            settings.DUMPED_USERS_DAYS_DELETE):
            log.debug('Deleting old tarball: {0}'.format(filepath))
            os.remove(filepath)
Esempio n. 34
0
def addon_reviews_ratings():
    """Update all add-on total_reviews and average/bayesian ratings."""
    addons = Addon.objects.values_list('id', flat=True)
    ts = [
        tasks.cron_review_aggregate.subtask(args=chunk)
        for chunk in chunked(addons, 100)
    ]
    TaskSet(ts).apply_async()
Esempio n. 35
0
def invalidate_users(**kw):
    """Invalidate all users to reflect latest Marketplace access whitelist."""
    from amo.utils import chunked
    from users.models import UserProfile
    log.info('Invalidating users for access whitelist.')
    d = UserProfile.objects.values_list('id', flat=True)
    ts = [_invalidate_users.subtask(args=[chunk]) for chunk in chunked(d, 100)]
    TaskSet(ts).apply_async()
Esempio n. 36
0
 def handle(self, *args, **options):
     task = tasks.get(options.get('task'))
     if not task:
         raise ValueError('Unknown task: %s' % ', '.join(tasks.keys()))
     pks = (Addon.objects.filter(*task['qs']).values_list(
         'pk', flat=True).order_by('id'))
     for chunk in chunked(pks, 100):
         task['method'].delay(chunk)
Esempio n. 37
0
def reviews_denorm():
    """Set is_latest and previous_count for all reviews."""
    pairs = list(set(Review.objects.values_list('addon', 'user')))
    ts = [
        tasks.update_denorm.subtask(args=chunk)
        for chunk in chunked(pairs, 50)
    ]
    TaskSet(ts).apply_async()
Esempio n. 38
0
def collection_meta():
    from . import tasks
    collections = Collection.objects.values_list('id', flat=True)
    ts = [
        tasks.cron_collection_meta.subtask(args=chunk)
        for chunk in chunked(collections, 1000)
    ]
    TaskSet(ts).apply_async()
Esempio n. 39
0
def update_addons_collections_downloads():
    """Update addons+collections download totals."""

    d = AddonCollectionCount.objects.values("addon", "collection").annotate(sum=Sum("count"))

    with establish_connection() as conn:
        for chunk in chunked(d, 600):
            tasks.update_addons_collections_downloads.apply_async(args=[chunk], connection=conn)
Esempio n. 40
0
def update_collections_total():
    """Update collections downloads totals."""

    d = CollectionCount.objects.values("collection_id").annotate(sum=Sum("count"))

    with establish_connection() as conn:
        for chunk in chunked(d, 1000):
            tasks.update_collections_total.apply_async(args=[chunk], connection=conn)
Esempio n. 41
0
def reindex_users(index=None):
    from . import tasks
    ids = UserProfile.objects.values_list('id', flat=True)
    taskset = [
        tasks.index_users.subtask(args=[chunk], kwargs=dict(index=index))
        for chunk in chunked(sorted(list(ids)), 150)
    ]
    TaskSet(taskset).apply_async()
Esempio n. 42
0
def run():
    """Delete duplicate image assets."""
    for chunk in chunked(Webapp.objects.all(), 50):
        for app in chunk:
            for slug in SIZE_SLUGS:
                assets = ImageAsset.objects.filter(addon=app, slug=slug)
                for asset in assets[1:]:
                    asset.delete()
Esempio n. 43
0
def update_addons_current_version():
    """Update the current_version field of the addons."""
    d = Addon.objects.valid().exclude(type=amo.ADDON_PERSONA).values_list('id')

    with establish_connection() as conn:
        for chunk in chunked(d, 1000):
            _update_addons_current_version.apply_async(args=[chunk],
                                                       connection=conn)
Esempio n. 44
0
def run():
    """Delete duplicate image assets."""
    for chunk in chunked(Webapp.objects.all(), 50):
        for app in chunk:
            for slug in SIZE_SLUGS:
                assets = ImageAsset.objects.filter(addon=app, slug=slug)
                for asset in assets[1:]:
                    asset.delete()
Esempio n. 45
0
def invalidate_users(**kw):
    """Invalidate all users to reflect latest Marketplace access whitelist."""
    from amo.utils import chunked
    from users.models import UserProfile
    log.info('Invalidating users for access whitelist.')
    d = UserProfile.objects.values_list('id', flat=True)
    ts = [_invalidate_users.subtask(args=[chunk]) for chunk in chunked(d, 100)]
    TaskSet(ts).apply_async()
    def handle(self, *args, **options):
        activity_ids = AppLog.objects.values_list('activity_log', flat=True)
        logs = (ActivityLog.objects.filter(
            pk__in=list(activity_ids), action__in=amo.LOG_REVIEW_QUEUE)
            .order_by('created'))

        for log_chunk in chunked(logs, 100):
            _migrate_activity_log.delay(log_chunk)
Esempio n. 47
0
def flip_webapp_status(from_, to):
    qs = Webapp.objects.filter(status=from_)
    # Grab the ids so we can get them reindexed.
    ids = list(qs.values_list('id', flat=True))
    qs.update(status=to)
    ts = [index_addons.subtask(args=[chunk]) for chunk in chunked(ids, 150)]
    # Delay these tasks to avoid slave lag.
    TaskSet(ts).apply_async(countdown=30)
Esempio n. 48
0
def update_all_appsupport():
    from .tasks import update_appsupport
    ids = sorted(set(AppSupport.objects.values_list('addon', flat=True)))
    task_log.info('Updating appsupport for %s addons.' % len(ids))
    for idx, chunk in enumerate(chunked(ids, 100)):
        if idx % 10 == 0:
            task_log.info('[%s/%s] Updating appsupport.'
                          % (idx * 100, len(ids)))
        update_appsupport(chunk)
Esempio n. 49
0
def update_collections_total():
    """Update collections downloads totals."""

    d = (CollectionCount.objects.values('collection_id')
                                .annotate(sum=Sum('count')))

    ts = [tasks.update_collections_total.subtask(args=[chunk])
          for chunk in chunked(d, 50)]
    TaskSet(ts).apply_async()
Esempio n. 50
0
def reindex_collections(index=None):
    from . import tasks
    ids = (Collection.objects.exclude(
        type=amo.COLLECTION_SYNCHRONIZED).values_list('id', flat=True))
    taskset = [
        tasks.index_collections.subtask(args=[chunk], kwargs=dict(index=index))
        for chunk in chunked(sorted(list(ids)), 150)
    ]
    TaskSet(taskset).apply_async()
Esempio n. 51
0
def featured_grid(context, addons, src=None, dl_src=None, pagesize=3, cols=3):
    if not src:
        src = context.get('src')
    # dl_src is an optional src paramater just for the download links
    if not dl_src:
        dl_src = src
    pages = chunked(addons, pagesize)
    columns = '' if cols != 3 else 'three-col'
    return new_context(**locals())
Esempio n. 52
0
def update_addons_collections_downloads():
    """Update addons+collections download totals."""

    d = (AddonCollectionCount.objects.values('addon', 'collection')
         .annotate(sum=Sum('count')))

    ts = [tasks.update_addons_collections_downloads.subtask(args=[chunk])
          for chunk in chunked(d, 600)]
    TaskSet(ts).apply_async()
Esempio n. 53
0
def update_collections_total():
    """Update collections downloads totals."""

    d = (CollectionCount.objects.values('collection_id')
                                .annotate(sum=Sum('count')))

    ts = [tasks.update_collections_total.subtask(args=[chunk])
          for chunk in chunked(d, 1000)]
    TaskSet(ts).apply_async()
Esempio n. 54
0
def update_all_appsupport():
    from .tasks import update_appsupport
    ids = sorted(set(AppSupport.objects.values_list('addon', flat=True)))
    task_log.info('Updating appsupport for %s addons.' % len(ids))
    for idx, chunk in enumerate(chunked(ids, 100)):
        if idx % 10 == 0:
            task_log.info('[%s/%s] Updating appsupport.' %
                          (idx * 100, len(ids)))
        update_appsupport(chunk)