def _get_monolith_jobs(date=None):
"""
Return a dict of Monolith based statistics queries.
The dict is of the form::
{'<metric_name>': [{'count': <callable>, 'dimensions': <dimensions>}]}
Where `dimensions` is an optional dict of dimensions we expect to filter on
via Monolith.
If a date is specified and applies to the job it will be used. Otherwise
the date will default to today().
"""
if not date:
date = datetime.date.today()
# If we have a datetime make it a date so H/M/S isn't used.
if isinstance(date, datetime.datetime):
date = date.date()
next_date = date + datetime.timedelta(days=1)
stats = {
# Marketplace reviews.
'apps_review_count_new': [{
'count': Review.objects.filter(
created__range=(date, next_date), editorreview=0,
addon__type=amo.ADDON_WEBAPP).count,
}],
# New users
'mmo_user_count_total': [{
'count': UserProfile.objects.filter(
created__lt=next_date,
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
}],
'mmo_user_count_new': [{
'count': UserProfile.objects.filter(
created__range=(date, next_date),
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
}],
# New developers.
'mmo_developer_count_total': [{
'count': AddonUser.objects.filter(
addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
}],
# App counts.
'apps_count_new': [{
'count': Addon.objects.filter(
created__range=(date, next_date), type=amo.ADDON_WEBAPP).count,
}],
'apps_count_installed': [{
'count': Installed.objects.filter(
created__range=(date, next_date),
addon__type=amo.ADDON_WEBAPP).count,
}],
}
# Add various "Apps Added" for all the dimensions we need.
apps = Addon.objects.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP)
package_counts = []
premium_counts = []
for region_slug, region in REGIONS_CHOICES_SLUG:
# Apps added by package type and region.
for package_type in ADDON_WEBAPP_TYPES.values():
package_counts.append({
'count': apps.filter(
is_packaged=package_type == 'packaged').exclude(
addonexcludedregion__region=region.id).count,
'dimensions': {'region': region_slug,
'package_type': package_type},
})
# Apps added by premium type and region.
for premium_type, pt_name in ADDON_PREMIUM_API.items():
premium_counts.append({
'count': apps.filter(
premium_type=premium_type).exclude(
addonexcludedregion__region=region.id).count,
'dimensions': {'region': region_slug,
'premium_type': pt_name},
})
stats.update({'apps_added_by_package_type': package_counts})
stats.update({'apps_added_by_premium_type': premium_counts})
return stats
def _get_monolith_jobs(date=None):
"""
Return a dict of Monolith based statistics queries.
The dict is of the form::
{'<metric_name>': [{'count': <callable>, 'dimensions': <dimensions>}]}
Where `dimensions` is an optional dict of dimensions we expect to filter on
via Monolith.
If a date is specified and applies to the job it will be used. Otherwise
the date will default to today().
"""
if not date:
date = datetime.date.today()
# If we have a datetime make it a date so H/M/S isn't used.
if isinstance(date, datetime.datetime):
date = date.date()
next_date = date + datetime.timedelta(days=1)
stats = {
# Marketplace reviews.
'apps_review_count_new': [{
'count': Review.objects.filter(
created__range=(date, next_date), editorreview=0,
addon__type=amo.ADDON_WEBAPP).count,
}],
# New users
'mmo_user_count_total': [{
'count': UserProfile.objects.filter(
created__lt=next_date,
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
}],
'mmo_user_count_new': [{
'count': UserProfile.objects.filter(
created__range=(date, next_date),
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
}],
# New developers.
'mmo_developer_count_total': [{
'count': AddonUser.objects.filter(
addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
}],
# App counts.
'apps_count_new': [{
'count': Addon.objects.filter(
created__range=(date, next_date), type=amo.ADDON_WEBAPP).count,
}],
'apps_count_installed': [{
'count': Installed.objects.filter(
created__range=(date, next_date),
addon__type=amo.ADDON_WEBAPP).count,
}],
}
# Add various "Apps Added" for all the dimensions we need.
apps = Addon.objects.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP)
package_counts = []
premium_counts = []
for region_slug, region in REGIONS_CHOICES_SLUG:
# Apps added by package type and region.
for package_type in ADDON_WEBAPP_TYPES.values():
package_counts.append({
'count': apps.filter(
is_packaged=package_type == 'packaged').exclude(
addonexcludedregion__region=region.id).count,
'dimensions': {'region': region_slug,
'package_type': package_type},
})
# Apps added by premium type and region.
for premium_type, pt_name in ADDON_PREMIUM_API.items():
premium_counts.append({
'count': apps.filter(
premium_type=premium_type).exclude(
addonexcludedregion__region=region.id).count,
'dimensions': {'region': region_slug,
'premium_type': pt_name},
})
stats.update({'apps_added_by_package_type': package_counts})
stats.update({'apps_added_by_premium_type': premium_counts})
return stats
def _get_daily_jobs(date=None):
"""Return a dictionary of statistics queries.
If a date is specified and applies to the job it will be used. Otherwise
the date will default to today().
"""
if not date:
date = datetime.date.today()
# Passing through a datetime would not generate an error,
# but would pass and give incorrect values.
if isinstance(date, datetime.datetime):
raise ValueError('This requires a valid date, not a datetime')
# Testing on lte created date doesn't get you todays date, you need to do
# less than next date. That's because 2012-1-1 becomes 2012-1-1 00:00
next_date = date + datetime.timedelta(days=1)
date_str = date.strftime('%Y-%m-%d')
extra = dict(where=['DATE(created)=%s'], params=[date_str])
# If you're editing these, note that you are returning a function! This
# cheesy hackery was done so that we could pass the queries to celery
# lazily and not hammer the db with a ton of these all at once.
stats = {
# Add-on Downloads
'addon_total_downloads': lambda: DownloadCount.objects.filter(
date__lt=next_date).aggregate(sum=Sum('count'))['sum'],
'addon_downloads_new': lambda: DownloadCount.objects.filter(
date=date).aggregate(sum=Sum('count'))['sum'],
# Add-on counts
'addon_count_new': Addon.objects.extra(**extra).count,
# Version counts
'version_count_new': Version.objects.extra(**extra).count,
# User counts
'user_count_total': UserProfile.objects.filter(
created__lt=next_date).count,
'user_count_new': UserProfile.objects.extra(**extra).count,
# Review counts
'review_count_total': Review.objects.filter(created__lte=date,
editorreview=0).count,
'review_count_new': Review.objects.filter(editorreview=0).extra(
**extra).count,
# Collection counts
'collection_count_total': Collection.objects.filter(
created__lt=next_date).count,
'collection_count_new': Collection.objects.extra(**extra).count,
'collection_count_autopublishers': Collection.objects.filter(
created__lt=next_date, type=amo.COLLECTION_SYNCHRONIZED).count,
'collection_addon_downloads': (lambda:
AddonCollectionCount.objects.filter(date__lte=date).aggregate(
sum=Sum('count'))['sum']),
# Marketplace stats
# TODO: Remove 'apps_count_new' once we fully migrate to the new
# 'apps_added_*' stats.
'apps_count_new': (Addon.objects
.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP).count),
'apps_count_installed': (Installed.objects
.filter(created__range=(date, next_date),
addon__type=amo.ADDON_WEBAPP).count),
# Marketplace reviews
'apps_review_count_new': Review.objects
.filter(created__range=(date, next_date),
editorreview=0, addon__type=amo.ADDON_WEBAPP).count,
# New users
'mmo_user_count_total': UserProfile.objects.filter(
created__lt=next_date,
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
'mmo_user_count_new': UserProfile.objects.filter(
created__range=(date, next_date),
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
# New developers
'mmo_developer_count_total': AddonUser.objects.filter(
addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
}
# Add various "Apps Added" for all the dimensions we need.
apps = Addon.objects.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP)
for region_slug, region in REGIONS_CHOICES_SLUG:
# Apps added by package type and region.
for package_type in ADDON_WEBAPP_TYPES.values():
stats.update({
'apps_added_%s_%s' % (region_slug, package_type):
apps.filter(is_packaged=package_type == 'packaged')
.exclude(addonexcludedregion__region=region.id).count
})
# Apps added by premium type and region.
for premium_type, pt_name in ADDON_PREMIUM_API.items():
stats.update({
'apps_added_%s_%s' % (region_slug, pt_name):
apps.filter(premium_type=premium_type)
.exclude(addonexcludedregion__region=region.id).count
})
# If we're processing today's stats, we'll do some extras. We don't do
# these for re-processed stats because they change over time (eg. add-ons
# move from sandbox -> public
if date == datetime.date.today():
stats.update({
'addon_count_experimental': Addon.objects.filter(
created__lte=date, status=amo.STATUS_UNREVIEWED,
disabled_by_user=0).count,
'addon_count_nominated': Addon.objects.filter(
created__lte=date, status=amo.STATUS_NOMINATED,
disabled_by_user=0).count,
'addon_count_public': Addon.objects.filter(
created__lte=date, status=amo.STATUS_PUBLIC,
disabled_by_user=0).count,
'addon_count_pending': Version.objects.filter(
created__lte=date, files__status=amo.STATUS_PENDING).count,
'collection_count_private': Collection.objects.filter(
created__lte=date, listed=0).count,
'collection_count_public': Collection.objects.filter(
created__lte=date, listed=1).count,
'collection_count_editorspicks': Collection.objects.filter(
created__lte=date, type=amo.COLLECTION_FEATURED).count,
'collection_count_normal': Collection.objects.filter(
created__lte=date, type=amo.COLLECTION_NORMAL).count,
})
return stats
# Map of URL metric name to monolith metric name.
#
# The 'dimensions' key is optional query string arguments with defaults that is
# passed to the monolith client and used in the facet filters. If the default
# is `None`, the dimension is excluded unless specified via the API.
#
# The 'lines' key is optional and used for multi-line charts. The format is:
# {'<name>': {'<dimension-key>': '<dimension-value>'}}
# where <name> is what's returned in the JSON output and the dimension
# key/value is what's sent to Monolith similar to the 'dimensions' above.
lines = lambda name, vals: dict((val, {name: val}) for val in vals)
STATS = {
'apps_added_by_package': {
'metric': 'apps_added_package_count',
'dimensions': {'region': 'us'},
'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
},
'apps_added_by_premium': {
'metric': 'apps_added_premium_count',
'dimensions': {'region': 'us'},
'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
},
'apps_available_by_package': {
'metric': 'apps_available_package_count',
'dimensions': {'region': 'us'},
'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
},
'apps_available_by_premium': {
'metric': 'apps_available_premium_count',
'dimensions': {'region': 'us'},
'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
def _get_daily_jobs(date=None):
"""Return a dictionary of statistics queries.
If a date is specified and applies to the job it will be used. Otherwise
the date will default to today().
"""
if not date:
date = datetime.date.today()
# Passing through a datetime would not generate an error,
# but would pass and give incorrect values.
if isinstance(date, datetime.datetime):
raise ValueError('This requires a valid date, not a datetime')
# Testing on lte created date doesn't get you todays date, you need to do
# less than next date. That's because 2012-1-1 becomes 2012-1-1 00:00
next_date = date + datetime.timedelta(days=1)
date_str = date.strftime('%Y-%m-%d')
extra = dict(where=['DATE(created)=%s'], params=[date_str])
# If you're editing these, note that you are returning a function! This
# cheesy hackery was done so that we could pass the queries to celery
# lazily and not hammer the db with a ton of these all at once.
stats = {
# Add-on Downloads
'addon_total_downloads':
lambda: DownloadCount.objects.filter(date__lt=next_date).aggregate(
sum=Sum('count'))['sum'],
'addon_downloads_new':
lambda: DownloadCount.objects.filter(date=date).aggregate(sum=Sum(
'count'))['sum'],
# Add-on counts
'addon_count_new':
Addon.objects.extra(**extra).count,
# Version counts
'version_count_new':
Version.objects.extra(**extra).count,
# User counts
'user_count_total':
UserProfile.objects.filter(created__lt=next_date).count,
'user_count_new':
UserProfile.objects.extra(**extra).count,
# Review counts
'review_count_total':
Review.objects.filter(created__lte=date, editorreview=0).count,
'review_count_new':
Review.objects.filter(editorreview=0).extra(**extra).count,
# Collection counts
'collection_count_total':
Collection.objects.filter(created__lt=next_date).count,
'collection_count_new':
Collection.objects.extra(**extra).count,
'collection_count_autopublishers':
Collection.objects.filter(created__lt=next_date,
type=amo.COLLECTION_SYNCHRONIZED).count,
'collection_addon_downloads':
(lambda: AddonCollectionCount.objects.filter(date__lte=date).aggregate(
sum=Sum('count'))['sum']),
# Marketplace stats
# TODO: Remove 'apps_count_new' once we fully migrate to the new
# 'apps_added_*' stats.
'apps_count_new':
(Addon.objects.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP).count),
'apps_count_installed':
(Installed.objects.filter(created__range=(date, next_date),
addon__type=amo.ADDON_WEBAPP).count),
# Marketplace reviews
'apps_review_count_new':
Review.objects.filter(created__range=(date, next_date),
editorreview=0,
addon__type=amo.ADDON_WEBAPP).count,
# New users
'mmo_user_count_total':
UserProfile.objects.filter(
created__lt=next_date,
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
'mmo_user_count_new':
UserProfile.objects.filter(
created__range=(date, next_date),
source=amo.LOGIN_SOURCE_MMO_BROWSERID).count,
# New developers
'mmo_developer_count_total':
AddonUser.objects.filter(
addon__type=amo.ADDON_WEBAPP).values('user').distinct().count,
}
# Add various "Apps Added" for all the dimensions we need.
apps = Addon.objects.filter(created__range=(date, next_date),
type=amo.ADDON_WEBAPP)
for region_slug, region in REGIONS_CHOICES_SLUG:
# Apps added by package type and region.
for package_type in ADDON_WEBAPP_TYPES.values():
stats.update({
'apps_added_%s_%s' % (region_slug, package_type):
apps.filter(is_packaged=package_type == 'packaged').exclude(
addonexcludedregion__region=region.id).count
})
# Apps added by premium type and region.
for premium_type, pt_name in ADDON_PREMIUM_API.items():
stats.update({
'apps_added_%s_%s' % (region_slug, pt_name):
apps.filter(premium_type=premium_type).exclude(
addonexcludedregion__region=region.id).count
})
# If we're processing today's stats, we'll do some extras. We don't do
# these for re-processed stats because they change over time (eg. add-ons
# move from sandbox -> public
if date == datetime.date.today():
stats.update({
'addon_count_experimental':
Addon.objects.filter(created__lte=date,
status=amo.STATUS_UNREVIEWED,
disabled_by_user=0).count,
'addon_count_nominated':
Addon.objects.filter(created__lte=date,
status=amo.STATUS_NOMINATED,
disabled_by_user=0).count,
'addon_count_public':
Addon.objects.filter(created__lte=date,
status=amo.STATUS_PUBLIC,
disabled_by_user=0).count,
'addon_count_pending':
Version.objects.filter(created__lte=date,
files__status=amo.STATUS_PENDING).count,
'collection_count_private':
Collection.objects.filter(created__lte=date, listed=0).count,
'collection_count_public':
Collection.objects.filter(created__lte=date, listed=1).count,
'collection_count_editorspicks':
Collection.objects.filter(created__lte=date,
type=amo.COLLECTION_FEATURED).count,
'collection_count_normal':
Collection.objects.filter(created__lte=date,
type=amo.COLLECTION_NORMAL).count,
})
return stats
# The 'dimensions' key is optional query string arguments with defaults that is
# passed to the monolith client and used in the facet filters. If the default
# is `None`, the dimension is excluded unless specified via the API.
#
# The 'lines' key is optional and used for multi-line charts. The format is:
# {'<name>': {'<dimension-key>': '<dimension-value>'}}
# where <name> is what's returned in the JSON output and the dimension
# key/value is what's sent to Monolith similar to the 'dimensions' above.
lines = lambda name, vals: dict((val, {name: val}) for val in vals)
STATS = {
'apps_added_by_package': {
'metric': 'apps_added_package_count',
'dimensions': {
'region': 'us'
},
'lines': lines('package_type', ADDON_WEBAPP_TYPES.values()),
},
'apps_added_by_premium': {
'metric': 'apps_added_premium_count',
'dimensions': {
'region': 'us'
},
'lines': lines('premium_type', ADDON_PREMIUM_API.values()),
},
'apps_installed': {
'metric': 'app_installs',
'dimensions': {
'region': None
},
},
'total_developers': {
