Esempio n. 1
0
def _get_monolith_jobs(date=None):
    """
    Return a dict of Monolith based statistics queries.

    The dict is of the form::

        {'<metric_name>': [{'count': <callable>, 'dimensions': <dimensions>}]}

    Where `dimensions` is an optional dict of dimensions we expect to filter on
    via Monolith.

    If a date is specified and applies to the job it will be used.  Otherwise
    the date will default to today().
    """
    if not date:
        date = datetime.date.today()

    # If we have a datetime make it a date so H/M/S isn't used.
    if isinstance(date, datetime.datetime):
        date = date.date()

    next_date = date + datetime.timedelta(days=1)

    stats = {
        # Marketplace reviews.
        'apps_review_count_new': [{
            'count': Review.objects.filter(
                created__range=(date, next_date), editorreview=0,
                addon__type=amo.ADDON_WEBAPP).count,
        }],

        # New users
        'mmo_user_count_total': [{
            'count': UserProfile.objects.filter(
                created__lt=next_date,
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        }],
        'mmo_user_count_new': [{
            'count': UserProfile.objects.filter(
                created__range=(date, next_date),
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        }],

        # New developers.
        'mmo_developer_count_total': [{
            'count': AddonUser.objects.filter(
                addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
        }],

        # App counts.
        'apps_count_new': [{
            'count': Addon.objects.filter(
                created__range=(date, next_date), type=amo.ADDON_WEBAPP).count,
        }],
        'apps_count_installed': [{
            'count': Installed.objects.filter(
                created__range=(date, next_date),
                addon__type=amo.ADDON_WEBAPP).count,
        }],
    }

    # Add various "Apps Added" for all the dimensions we need.
    apps = Addon.objects.filter(created__range=(date, next_date),
                                type=amo.ADDON_WEBAPP)
    package_counts = []
    premium_counts = []

    for region_slug, region in REGIONS_CHOICES_SLUG:
        # Apps added by package type and region.
        for package_type in ADDON_WEBAPP_TYPES.values():
            package_counts.append({
                'count': apps.filter(
                    is_packaged=package_type == 'packaged').exclude(
                        addonexcludedregion__region=region.id).count,
                'dimensions': {'region': region_slug,
                               'package_type': package_type},
            })

        # Apps added by premium type and region.
        for premium_type, pt_name in ADDON_PREMIUM_API.items():
            premium_counts.append({
                'count': apps.filter(
                    premium_type=premium_type).exclude(
                        addonexcludedregion__region=region.id).count,
                'dimensions': {'region': region_slug,
                               'premium_type': pt_name},
            })

    stats.update({'apps_added_by_package_type': package_counts})
    stats.update({'apps_added_by_premium_type': premium_counts})

    return stats
Esempio n. 2
0
def _get_monolith_jobs(date=None):
    """
    Return a dict of Monolith based statistics queries.

    The dict is of the form::

        {'<metric_name>': [{'count': <callable>, 'dimensions': <dimensions>}]}

    Where `dimensions` is an optional dict of dimensions we expect to filter on
    via Monolith.

    If a date is specified and applies to the job it will be used.  Otherwise
    the date will default to today().
    """
    if not date:
        date = datetime.date.today()

    # If we have a datetime make it a date so H/M/S isn't used.
    if isinstance(date, datetime.datetime):
        date = date.date()

    next_date = date + datetime.timedelta(days=1)

    stats = {
        # Marketplace reviews.
        'apps_review_count_new': [{
            'count': Review.objects.filter(
                created__range=(date, next_date), editorreview=0,
                addon__type=amo.ADDON_WEBAPP).count,
        }],

        # New users
        'mmo_user_count_total': [{
            'count': UserProfile.objects.filter(
                created__lt=next_date,
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        }],
        'mmo_user_count_new': [{
            'count': UserProfile.objects.filter(
                created__range=(date, next_date),
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        }],

        # New developers.
        'mmo_developer_count_total': [{
            'count': AddonUser.objects.filter(
                addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
        }],

        # App counts.
        'apps_count_new': [{
            'count': Addon.objects.filter(
                created__range=(date, next_date), type=amo.ADDON_WEBAPP).count,
        }],
        'apps_count_installed': [{
            'count': Installed.objects.filter(
                created__range=(date, next_date),
                addon__type=amo.ADDON_WEBAPP).count,
        }],
    }

    # Add various "Apps Added" for all the dimensions we need.
    apps = Addon.objects.filter(created__range=(date, next_date),
                                type=amo.ADDON_WEBAPP)
    package_counts = []
    premium_counts = []

    for region_slug, region in REGIONS_CHOICES_SLUG:
        # Apps added by package type and region.
        for package_type in ADDON_WEBAPP_TYPES.values():
            package_counts.append({
                'count': apps.filter(
                    is_packaged=package_type == 'packaged').exclude(
                        addonexcludedregion__region=region.id).count,
                'dimensions': {'region': region_slug,
                               'package_type': package_type},
            })

        # Apps added by premium type and region.
        for premium_type, pt_name in ADDON_PREMIUM_API.items():
            premium_counts.append({
                'count': apps.filter(
                    premium_type=premium_type).exclude(
                        addonexcludedregion__region=region.id).count,
                'dimensions': {'region': region_slug,
                               'premium_type': pt_name},
            })

    stats.update({'apps_added_by_package_type': package_counts})
    stats.update({'apps_added_by_premium_type': premium_counts})

    return stats
Esempio n. 3
0
def _get_daily_jobs(date=None):
    """Return a dictionary of statistics queries.

    If a date is specified and applies to the job it will be used.  Otherwise
    the date will default to today().
    """
    if not date:
        date = datetime.date.today()

    # Passing through a datetime would not generate an error,
    # but would pass and give incorrect values.
    if isinstance(date, datetime.datetime):
        raise ValueError('This requires a valid date, not a datetime')

    # Testing on lte created date doesn't get you todays date, you need to do
    # less than next date. That's because 2012-1-1 becomes 2012-1-1 00:00
    next_date = date + datetime.timedelta(days=1)

    date_str = date.strftime('%Y-%m-%d')
    extra = dict(where=['DATE(created)=%s'], params=[date_str])

    # If you're editing these, note that you are returning a function!  This
    # cheesy hackery was done so that we could pass the queries to celery
    # lazily and not hammer the db with a ton of these all at once.
    stats = {
        # Add-on Downloads
        'addon_total_downloads': lambda: DownloadCount.objects.filter(
                date__lt=next_date).aggregate(sum=Sum('count'))['sum'],
        'addon_downloads_new': lambda: DownloadCount.objects.filter(
                date=date).aggregate(sum=Sum('count'))['sum'],

        # Add-on counts
        'addon_count_new': Addon.objects.extra(**extra).count,

        # Version counts
        'version_count_new': Version.objects.extra(**extra).count,

        # User counts
        'user_count_total': UserProfile.objects.filter(
                created__lt=next_date).count,
        'user_count_new': UserProfile.objects.extra(**extra).count,

        # Review counts
        'review_count_total': Review.objects.filter(created__lte=date,
                                                    editorreview=0).count,
        'review_count_new': Review.objects.filter(editorreview=0).extra(
                **extra).count,

        # Collection counts
        'collection_count_total': Collection.objects.filter(
                created__lt=next_date).count,
        'collection_count_new': Collection.objects.extra(**extra).count,
        'collection_count_autopublishers': Collection.objects.filter(
                created__lt=next_date, type=amo.COLLECTION_SYNCHRONIZED).count,

        'collection_addon_downloads': (lambda:
            AddonCollectionCount.objects.filter(date__lte=date).aggregate(
                sum=Sum('count'))['sum']),

        # Marketplace stats
        # TODO: Remove 'apps_count_new' once we fully migrate to the new
        # 'apps_added_*' stats.
        'apps_count_new': (Addon.objects
                .filter(created__range=(date, next_date),
                        type=amo.ADDON_WEBAPP).count),
        'apps_count_installed': (Installed.objects
                .filter(created__range=(date, next_date),
                        addon__type=amo.ADDON_WEBAPP).count),

        # Marketplace reviews
        'apps_review_count_new': Review.objects
                .filter(created__range=(date, next_date),
                        editorreview=0, addon__type=amo.ADDON_WEBAPP).count,

        # New users
        'mmo_user_count_total': UserProfile.objects.filter(
                created__lt=next_date,
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        'mmo_user_count_new': UserProfile.objects.filter(
                created__range=(date, next_date),
                source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,

        # New developers
        'mmo_developer_count_total': AddonUser.objects.filter(
            addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
    }

    # Add various "Apps Added" for all the dimensions we need.
    apps = Addon.objects.filter(created__range=(date, next_date),
                                type=amo.ADDON_WEBAPP)
    for region_slug, region in REGIONS_CHOICES_SLUG:
        # Apps added by package type and region.
        for package_type in ADDON_WEBAPP_TYPES.values():
            stats.update({
                'apps_added_%s_%s' % (region_slug, package_type):
                apps.filter(is_packaged=package_type == 'packaged')
                    .exclude(addonexcludedregion__region=region.id).count
            })
        # Apps added by premium type and region.
        for premium_type, pt_name in ADDON_PREMIUM_API.items():
            stats.update({
                'apps_added_%s_%s' % (region_slug, pt_name):
                apps.filter(premium_type=premium_type)
                    .exclude(addonexcludedregion__region=region.id).count
            })

    # If we're processing today's stats, we'll do some extras.  We don't do
    # these for re-processed stats because they change over time (eg. add-ons
    # move from sandbox -> public
    if date == datetime.date.today():
        stats.update({
            'addon_count_experimental': Addon.objects.filter(
                created__lte=date, status=amo.STATUS_UNREVIEWED,
                disabled_by_user=0).count,
            'addon_count_nominated': Addon.objects.filter(
                created__lte=date, status=amo.STATUS_NOMINATED,
                disabled_by_user=0).count,
            'addon_count_public': Addon.objects.filter(
                created__lte=date, status=amo.STATUS_PUBLIC,
                disabled_by_user=0).count,
            'addon_count_pending': Version.objects.filter(
                created__lte=date, files__status=amo.STATUS_PENDING).count,

            'collection_count_private': Collection.objects.filter(
                created__lte=date, listed=0).count,
            'collection_count_public': Collection.objects.filter(
                created__lte=date, listed=1).count,
            'collection_count_editorspicks': Collection.objects.filter(
                created__lte=date, type=amo.COLLECTION_FEATURED).count,
            'collection_count_normal': Collection.objects.filter(
                created__lte=date, type=amo.COLLECTION_NORMAL).count,
        })

    return stats
Esempio n. 4
0
#
# The 'lines' key is optional and used for multi-line charts. The format is:
#     {'<name>': {'<dimension-key>': '<dimension-value>'}}
# where <name> is what's returned in the JSON output and the dimension
# key/value is what's sent to Monolith similar to the 'dimensions' above.
lines = lambda name, vals: dict((val, {name: val}) for val in vals)
STATS = {
    'apps_added_by_package': {
        'metric': 'apps_added_package_count',
        'dimensions': {'region': 'us'},
        'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
    },
    'apps_added_by_premium': {
        'metric': 'apps_added_premium_count',
        'dimensions': {'region': 'us'},
        'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
    },
    'apps_available_by_package': {
        'metric': 'apps_available_package_count',
        'dimensions': {'region': 'us'},
        'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
    },
    'apps_available_by_premium': {
        'metric': 'apps_available_premium_count',
        'dimensions': {'region': 'us'},
        'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
    },
    'apps_installed': {
        'metric': 'app_installs',
        'dimensions': {'region': None},
    },
Esempio n. 5
0
def _get_daily_jobs(date=None):
    """Return a dictionary of statistics queries.

    If a date is specified and applies to the job it will be used.  Otherwise
    the date will default to today().
    """
    if not date:
        date = datetime.date.today()

    # Passing through a datetime would not generate an error,
    # but would pass and give incorrect values.
    if isinstance(date, datetime.datetime):
        raise ValueError('This requires a valid date, not a datetime')

    # Testing on lte created date doesn't get you todays date, you need to do
    # less than next date. That's because 2012-1-1 becomes 2012-1-1 00:00
    next_date = date + datetime.timedelta(days=1)

    date_str = date.strftime('%Y-%m-%d')
    extra = dict(where=['DATE(created)=%s'], params=[date_str])

    # If you're editing these, note that you are returning a function!  This
    # cheesy hackery was done so that we could pass the queries to celery
    # lazily and not hammer the db with a ton of these all at once.
    stats = {
        # Add-on Downloads
        'addon_total_downloads':
        lambda: DownloadCount.objects.filter(date__lt=next_date).aggregate(
            sum=Sum('count'))['sum'],
        'addon_downloads_new':
        lambda: DownloadCount.objects.filter(date=date).aggregate(sum=Sum(
            'count'))['sum'],

        # Add-on counts
        'addon_count_new':
        Addon.objects.extra(**extra).count,

        # Version counts
        'version_count_new':
        Version.objects.extra(**extra).count,

        # User counts
        'user_count_total':
        UserProfile.objects.filter(created__lt=next_date).count,
        'user_count_new':
        UserProfile.objects.extra(**extra).count,

        # Review counts
        'review_count_total':
        Review.objects.filter(created__lte=date, editorreview=0).count,
        'review_count_new':
        Review.objects.filter(editorreview=0).extra(**extra).count,

        # Collection counts
        'collection_count_total':
        Collection.objects.filter(created__lt=next_date).count,
        'collection_count_new':
        Collection.objects.extra(**extra).count,
        'collection_count_autopublishers':
        Collection.objects.filter(created__lt=next_date,
                                  type=amo.COLLECTION_SYNCHRONIZED).count,
        'collection_addon_downloads':
        (lambda: AddonCollectionCount.objects.filter(date__lte=date).aggregate(
            sum=Sum('count'))['sum']),

        # Marketplace stats
        # TODO: Remove 'apps_count_new' once we fully migrate to the new
        # 'apps_added_*' stats.
        'apps_count_new':
        (Addon.objects.filter(created__range=(date, next_date),
                              type=amo.ADDON_WEBAPP).count),
        'apps_count_installed':
        (Installed.objects.filter(created__range=(date, next_date),
                                  addon__type=amo.ADDON_WEBAPP).count),

        # Marketplace reviews
        'apps_review_count_new':
        Review.objects.filter(created__range=(date, next_date),
                              editorreview=0,
                              addon__type=amo.ADDON_WEBAPP).count,

        # New users
        'mmo_user_count_total':
        UserProfile.objects.filter(
            created__lt=next_date,
            source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
        'mmo_user_count_new':
        UserProfile.objects.filter(
            created__range=(date, next_date),
            source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,

        # New developers
        'mmo_developer_count_total':
        AddonUser.objects.filter(
            addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
    }

    # Add various "Apps Added" for all the dimensions we need.
    apps = Addon.objects.filter(created__range=(date, next_date),
                                type=amo.ADDON_WEBAPP)
    for region_slug, region in REGIONS_CHOICES_SLUG:
        # Apps added by package type and region.
        for package_type in ADDON_WEBAPP_TYPES.values():
            stats.update({
                'apps_added_%s_%s' % (region_slug, package_type):
                apps.filter(is_packaged=package_type == 'packaged').exclude(
                    addonexcludedregion__region=region.id).count
            })
        # Apps added by premium type and region.
        for premium_type, pt_name in ADDON_PREMIUM_API.items():
            stats.update({
                'apps_added_%s_%s' % (region_slug, pt_name):
                apps.filter(premium_type=premium_type).exclude(
                    addonexcludedregion__region=region.id).count
            })

    # If we're processing today's stats, we'll do some extras.  We don't do
    # these for re-processed stats because they change over time (eg. add-ons
    # move from sandbox -> public
    if date == datetime.date.today():
        stats.update({
            'addon_count_experimental':
            Addon.objects.filter(created__lte=date,
                                 status=amo.STATUS_UNREVIEWED,
                                 disabled_by_user=0).count,
            'addon_count_nominated':
            Addon.objects.filter(created__lte=date,
                                 status=amo.STATUS_NOMINATED,
                                 disabled_by_user=0).count,
            'addon_count_public':
            Addon.objects.filter(created__lte=date,
                                 status=amo.STATUS_PUBLIC,
                                 disabled_by_user=0).count,
            'addon_count_pending':
            Version.objects.filter(created__lte=date,
                                   files__status=amo.STATUS_PENDING).count,
            'collection_count_private':
            Collection.objects.filter(created__lte=date, listed=0).count,
            'collection_count_public':
            Collection.objects.filter(created__lte=date, listed=1).count,
            'collection_count_editorspicks':
            Collection.objects.filter(created__lte=date,
                                      type=amo.COLLECTION_FEATURED).count,
            'collection_count_normal':
            Collection.objects.filter(created__lte=date,
                                      type=amo.COLLECTION_NORMAL).count,
        })

    return stats
Esempio n. 6
0
# key/value is what's sent to Monolith similar to the 'dimensions' above.
lines = lambda name, vals: dict((val, {name: val}) for val in vals)
STATS = {
    'apps_added_by_package': {
        'metric': 'apps_added_package_count',
        'dimensions': {
            'region': 'us'
        },
        'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
    },
    'apps_added_by_premium': {
        'metric': 'apps_added_premium_count',
        'dimensions': {
            'region': 'us'
        },
        'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
    },
    'apps_installed': {
        'metric': 'app_installs',
        'dimensions': {
            'region': None
        },
    },
    'total_developers': {
        'metric': 'total_dev_count'
    },
    'total_visits': {
        'metric': 'visits'
    },
}