Example #1
0
def _get_trending(app_id, region=None):
    """
    Calculate trending.

    a = installs from 7 days ago to now
    b = installs from 28 days ago to 8 days ago, averaged per week

    trending = (a - b) / b if a > 100 and b > 1 else 0

    """
    client = get_monolith_client()

    kwargs = {"app-id": app_id}
    if region:
        kwargs["region"] = region.slug

    today = datetime.datetime.today()

    # If we query monolith with interval=week and the past 7 days
    # crosses a Monday, Monolith splits the counts into two. We want
    # the sum over the past week so we need to `sum` these.
    try:
        count_1 = sum(
            c["count"] for c in client("app_installs", days_ago(7), today, "week", **kwargs) if c.get("count")
        )
    except ValueError as e:
        task_log.info("Call to ES failed: {0}".format(e))
        count_1 = 0

    # If count_1 isn't more than 100, stop here to avoid extra Monolith calls.
    if not count_1 > 100:
        return 0.0

    # Get the average installs for the prior 3 weeks. Don't use the `len` of
    # the returned counts because of week boundaries.
    try:
        count_3 = (
            sum(
                c["count"]
                for c in client("app_installs", days_ago(28), days_ago(8), "week", **kwargs)
                if c.get("count")
            )
            / 3
        )
    except ValueError as e:
        task_log.info("Call to ES failed: {0}".format(e))
        count_3 = 0

    if count_3 > 1:
        return (count_1 - count_3) / count_3
    else:
        return 0.0
Example #2
0
def _get_trending(app_id, region=None):
    """
    Calculate trending.

    a = installs from 7 days ago to now
    b = installs from 28 days ago to 8 days ago, averaged per week

    trending = (a - b) / b if a > 100 and b > 1 else 0

    """
    client = get_monolith_client()

    kwargs = {'app-id': app_id}
    if region:
        kwargs['region'] = region.slug

    today = datetime.datetime.today()

    # If we query monolith with interval=week and the past 7 days
    # crosses a Monday, Monolith splits the counts into two. We want
    # the sum over the past week so we need to `sum` these.
    try:
        count_1 = sum(
            c['count'] for c in
            client('app_installs', days_ago(7), today, 'week', **kwargs)
            if c.get('count'))
    except ValueError as e:
        task_log.info('Call to ES failed: {0}'.format(e))
        count_1 = 0

    # If count_1 isn't more than 100, stop here to avoid extra Monolith calls.
    if not count_1 > 100:
        return 0.0

    # Get the average installs for the prior 3 weeks. Don't use the `len` of
    # the returned counts because of week boundaries.
    try:
        count_3 = sum(
            c['count'] for c in
            client('app_installs', days_ago(28), days_ago(8), 'week', **kwargs)
            if c.get('count')) / 3
    except ValueError as e:
        task_log.info('Call to ES failed: {0}'.format(e))
        count_3 = 0

    if count_3 > 1:
        return (count_1 - count_3) / count_3
    else:
        return 0.0
Example #3
0
def update_downloads(ids, **kw):
    client = get_monolith_client()
    today = datetime.date.today()
    count = 0

    for app in Webapp.objects.filter(id__in=ids).no_transforms():

        kwargs = {'app-id': app.id}

        # Get weekly downloads.
        #
        # If we query monolith with interval=week and the past 7 days
        # crosses a Monday, Monolith splits the counts into two. We want
        # the sum over the past week so we need to `sum` these.
        try:
            weekly = sum(
                c['count'] for c in
                client('app_installs', days_ago(7).date(), today, 'week',
                       **kwargs)
                if c.get('count'))
        except ValueError as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            weekly = 0

        # Get total downloads.
        #
        # The monolith client lib doesn't handle this for us so we send a raw
        # ES query to Monolith.
        query = {'query': {'match_all': {}},
                 'facets': {
                     'installs': {
                         'statistical': {'field': 'app_installs'},
                         'facet_filter': {'term': kwargs}}},
                 'size': 0}
        try:
            resp = client.raw(query)
            total = resp.get('facets', {}).get('installs', {}).get('total', 0)
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            total = 0

        # Update Webapp object, if needed.
        update = False
        signal = False
        if weekly != app.weekly_downloads:
            update = True
            signal = True
        if total != app.total_downloads:
            update = True

        if update:
            # Note: Calling `update` will trigger a reindex on the app if
            # `_signal` is True. Since we only index `weekly_downloads`, we
            # can skip reindexing if this hasn't changed.
            count += 1
            app.update(weekly_downloads=weekly, total_downloads=total,
                       _signal=signal)

    task_log.info('App downloads updated for %s out of %s apps.'
                  % (count, len(ids)))
Example #4
0
def _app_downloads(app):
    stats = {"last_7_days": 0, "last_24_hours": 0, "alltime": 0}

    if app.is_webapp():
        # Webapps populate these fields via Monolith.
        stats["last_24_hours"] = "N/A"
        stats["last_7_days"] = app.weekly_downloads
        stats["alltime"] = app.total_downloads
        return stats

    qs = DownloadCount.objects.filter(addon=app)

    def sum_(qs):
        return qs.aggregate(total=Sum("count"))["total"] or 0

    stats["last_24_hours"] = sum_(qs.filter(date__gt=days_ago(1).date()))
    stats["last_7_days"] = sum_(qs.filter(date__gte=days_ago(7).date()))
    stats["alltime"] = sum_(qs)

    return stats
Example #5
0
def _app_downloads(app):
    stats = {'last_7_days': 0, 'last_24_hours': 0, 'alltime': 0}

    if app.is_webapp():
        # Webapps populate these fields via Monolith.
        stats['last_24_hours'] = 'N/A'
        stats['last_7_days'] = app.weekly_downloads
        stats['alltime'] = app.total_downloads
        return stats

    qs = DownloadCount.objects.filter(addon=app)

    def sum_(qs):
        return qs.aggregate(total=Sum('count'))['total'] or 0

    stats['last_24_hours'] = sum_(qs.filter(date__gt=days_ago(1).date()))
    stats['last_7_days'] = sum_(qs.filter(date__gte=days_ago(7).date()))
    stats['alltime'] = sum_(qs)

    return stats
Example #6
0
def _app_downloads(app):
    stats = {'last_7_days': 0,
             'last_24_hours': 0,
             'alltime': 0}

    if app.is_webapp():
        # Webapps populate these fields via Monolith.
        stats['last_24_hours'] = 'N/A'
        stats['last_7_days'] = app.weekly_downloads
        stats['alltime'] = app.total_downloads
        return stats

    qs = DownloadCount.objects.filter(addon=app)

    def sum_(qs):
        return qs.aggregate(total=Sum('count'))['total'] or 0

    stats['last_24_hours'] = sum_(qs.filter(date__gt=days_ago(1).date()))
    stats['last_7_days'] = sum_(qs.filter(date__gte=days_ago(7).date()))
    stats['alltime'] = sum_(qs)

    return stats
Example #7
0
def _weekly_theme_counts():
    """Returns unreviewed themes progress."""
    base_filters = {
        'pending_themes': Addon.objects.filter(
            type=amo.ADDON_PERSONA, status=amo.STATUS_PENDING),
        'flagged_themes': Addon.objects.filter(
            type=amo.ADDON_PERSONA, status=amo.STATUS_REVIEW_PENDING),
        'rereview_themes': RereviewQueueTheme.objects.all(),
    }

    theme_counts = {}
    for queue_type, qs in base_filters.iteritems():
        theme_counts[queue_type] = {
            'week': qs.filter(created__gte=days_ago(7)).count()
        }

    return theme_counts
Example #8
0
def _weekly_theme_counts():
    """Returns unreviewed themes progress."""
    base_filters = {
        'pending_themes': Addon.objects.filter(
            type=amo.ADDON_PERSONA, status=amo.STATUS_PENDING),
        'flagged_themes': Addon.objects.filter(
            type=amo.ADDON_PERSONA, status=amo.STATUS_REVIEW_PENDING),
        'rereview_themes': RereviewQueueTheme.objects.all(),
    }

    theme_counts = {}
    for queue_type, qs in base_filters.iteritems():
        theme_counts[queue_type] = {
            'week': qs.filter(created__gte=days_ago(7)).count()
        }

    return theme_counts
Example #9
0
def update_downloads(ids, **kw):
    client = get_monolith_client()
    count = 0

    for app in Webapp.objects.filter(id__in=ids).no_transforms():

        appid = {'app-id': app.id}

        # Get weekly downloads.
        query = {
            'query': {
                'match_all': {}
            },
            'facets': {
                'installs': {
                    'date_histogram': {
                        'value_field': 'app_installs',
                        'interval': 'week',
                        'key_field': 'date',
                    },
                    'facet_filter': {
                        'and':
                        [{
                            'term': appid
                        },
                         {
                             'range': {
                                 'date': {
                                     'gte':
                                     days_ago(8).date().strftime('%Y-%m-%d'),
                                     'lte':
                                     days_ago(1).date().strftime('%Y-%m-%d'),
                                 }
                             }
                         }]
                    }
                }
            },
            'size': 0
        }

        try:
            resp = client.raw(query)
            # If we query monolith with interval=week and the past 7 days
            # crosses a Monday, Monolith splits the counts into two. We want
            # the sum over the past week so we need to `sum` these.
            weekly = sum(c['total'] for c in resp.get('facets', {}).get(
                'installs', {}).get('entries') if c.get('total'))
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            weekly = 0

        # Get total downloads.
        query = {
            'query': {
                'match_all': {}
            },
            'facets': {
                'installs': {
                    'statistical': {
                        'field': 'app_installs'
                    },
                    'facet_filter': {
                        'term': appid
                    }
                }
            },
            'size': 0
        }
        try:
            resp = client.raw(query)
            total = resp.get('facets', {}).get('installs', {}).get('total', 0)
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            total = 0

        # Update Webapp object, if needed.
        update = False
        signal = False
        if weekly != app.weekly_downloads:
            update = True
            signal = True
        if total != app.total_downloads:
            update = True

        if update:
            # Note: Calling `update` will trigger a reindex on the app if
            # `_signal` is True. Since we only index `weekly_downloads`, we
            # can skip reindexing if this hasn't changed.
            count += 1
            app.update(weekly_downloads=weekly,
                       total_downloads=total,
                       _signal=signal)

    task_log.info('App downloads updated for %s out of %s apps.' %
                  (count, len(ids)))
Example #10
0
def update_downloads(ids, **kw):
    client = get_monolith_client()
    count = 0

    for app in Webapp.objects.filter(id__in=ids).no_transforms():

        appid = {'app-id': app.id}

        # Get weekly downloads.
        query = {
            'query': {'match_all': {}},
            'facets': {
                'installs': {
                    'date_histogram': {
                        'value_field': 'app_installs',
                        'interval': 'week',
                        'key_field': 'date',
                    },
                    'facet_filter': {
                        'and': [
                            {'term': appid},
                            {'range': {'date': {
                                'gte': days_ago(8).date().strftime('%Y-%m-%d'),
                                'lte': days_ago(1).date().strftime('%Y-%m-%d'),
                            }}}
                        ]
                    }
                }
            },
            'size': 0}

        try:
            resp = client.raw(query)
            # If we query monolith with interval=week and the past 7 days
            # crosses a Monday, Monolith splits the counts into two. We want
            # the sum over the past week so we need to `sum` these.
            weekly = sum(
                c['total'] for c in
                resp.get('facets', {}).get('installs', {}).get('entries')
                if c.get('total'))
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            weekly = 0

        # Get total downloads.
        query = {'query': {'match_all': {}},
                 'facets': {
                     'installs': {
                         'statistical': {'field': 'app_installs'},
                         'facet_filter': {'term': appid}}},
                 'size': 0}
        try:
            resp = client.raw(query)
            total = resp.get('facets', {}).get('installs', {}).get('total', 0)
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            total = 0

        # Update Webapp object, if needed.
        update = False
        signal = False
        if weekly != app.weekly_downloads:
            update = True
            signal = True
        if total != app.total_downloads:
            update = True

        if update:
            # Note: Calling `update` will trigger a reindex on the app if
            # `_signal` is True. Since we only index `weekly_downloads`, we
            # can skip reindexing if this hasn't changed.
            count += 1
            app.update(weekly_downloads=weekly, total_downloads=total,
                       _signal=signal)

    task_log.info('App downloads updated for %s out of %s apps.'
                  % (count, len(ids)))
Example #11
0
def update_downloads(ids, **kw):
    client = get_monolith_client()
    today = datetime.date.today()
    count = 0

    for app in Webapp.objects.filter(id__in=ids).no_transforms():

        kwargs = {'app-id': app.id}

        # Get weekly downloads.
        #
        # If we query monolith with interval=week and the past 7 days
        # crosses a Monday, Monolith splits the counts into two. We want
        # the sum over the past week so we need to `sum` these.
        try:
            weekly = sum(
                c['count']
                for c in client('app_installs',
                                days_ago(7).date(), today, 'week', **kwargs)
                if c.get('count'))
        except ValueError as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            weekly = 0

        # Get total downloads.
        #
        # The monolith client lib doesn't handle this for us so we send a raw
        # ES query to Monolith.
        query = {
            'query': {
                'match_all': {}
            },
            'facets': {
                'installs': {
                    'statistical': {
                        'field': 'app_installs'
                    },
                    'facet_filter': {
                        'term': kwargs
                    }
                }
            },
            'size': 0
        }
        try:
            resp = client.raw(query)
            total = resp.get('facets', {}).get('installs', {}).get('total', 0)
        except Exception as e:
            task_log.info('Call to ES failed: {0}'.format(e))
            total = 0

        # Update Webapp object, if needed.
        update = False
        signal = False
        if weekly != app.weekly_downloads:
            update = True
            signal = True
        if total != app.total_downloads:
            update = True

        if update:
            # Note: Calling `update` will trigger a reindex on the app if
            # `_signal` is True. Since we only index `weekly_downloads`, we
            # can skip reindexing if this hasn't changed.
            count += 1
            app.update(weekly_downloads=weekly,
                       total_downloads=total,
                       _signal=signal)

    task_log.info('App downloads updated for %s out of %s apps.' %
                  (count, len(ids)))
Example #12
0
def update_downloads(ids, **kw):
    client = get_monolith_client()
    count = 0

    for app in Webapp.objects.filter(id__in=ids).no_transforms():

        appid = {"app-id": app.id}

        # Get weekly downloads.
        query = {
            "query": {"match_all": {}},
            "facets": {
                "installs": {
                    "date_histogram": {"value_field": "app_installs", "interval": "week", "key_field": "date"},
                    "facet_filter": {
                        "and": [
                            {"term": appid},
                            {
                                "range": {
                                    "date": {
                                        "gte": days_ago(8).date().strftime("%Y-%m-%d"),
                                        "lte": days_ago(1).date().strftime("%Y-%m-%d"),
                                    }
                                }
                            },
                        ]
                    },
                }
            },
            "size": 0,
        }

        try:
            resp = client.raw(query)
            # If we query monolith with interval=week and the past 7 days
            # crosses a Monday, Monolith splits the counts into two. We want
            # the sum over the past week so we need to `sum` these.
            weekly = sum(
                c["total"] for c in resp.get("facets", {}).get("installs", {}).get("entries") if c.get("total")
            )
        except Exception as e:
            task_log.info("Call to ES failed: {0}".format(e))
            weekly = 0

        # Get total downloads.
        query = {
            "query": {"match_all": {}},
            "facets": {"installs": {"statistical": {"field": "app_installs"}, "facet_filter": {"term": appid}}},
            "size": 0,
        }
        try:
            resp = client.raw(query)
            total = resp.get("facets", {}).get("installs", {}).get("total", 0)
        except Exception as e:
            task_log.info("Call to ES failed: {0}".format(e))
            total = 0

        # Update Webapp object, if needed.
        update = False
        signal = False
        if weekly != app.weekly_downloads:
            update = True
            signal = True
        if total != app.total_downloads:
            update = True

        if update:
            # Note: Calling `update` will trigger a reindex on the app if
            # `_signal` is True. Since we only index `weekly_downloads`, we
            # can skip reindexing if this hasn't changed.
            count += 1
            app.update(weekly_downloads=weekly, total_downloads=total, _signal=signal)

    task_log.info("App downloads updated for %s out of %s apps." % (count, len(ids)))