def _get_trending(app_id, region=None):
"""
Calculate trending.
a = installs from 7 days ago to now
b = installs from 28 days ago to 8 days ago, averaged per week
trending = (a - b) / b if a > 100 and b > 1 else 0
"""
client = get_monolith_client()
kwargs = {"app-id": app_id}
if region:
kwargs["region"] = region.slug
today = datetime.datetime.today()
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
try:
count_1 = sum(
c["count"] for c in client("app_installs", days_ago(7), today, "week", **kwargs) if c.get("count")
)
except ValueError as e:
task_log.info("Call to ES failed: {0}".format(e))
count_1 = 0
# If count_1 isn't more than 100, stop here to avoid extra Monolith calls.
if not count_1 > 100:
return 0.0
# Get the average installs for the prior 3 weeks. Don't use the `len` of
# the returned counts because of week boundaries.
try:
count_3 = (
sum(
c["count"]
for c in client("app_installs", days_ago(28), days_ago(8), "week", **kwargs)
if c.get("count")
)
/ 3
)
except ValueError as e:
task_log.info("Call to ES failed: {0}".format(e))
count_3 = 0
if count_3 > 1:
return (count_1 - count_3) / count_3
else:
return 0.0
def _get_trending(app_id, region=None):
"""
Calculate trending.
a = installs from 7 days ago to now
b = installs from 28 days ago to 8 days ago, averaged per week
trending = (a - b) / b if a > 100 and b > 1 else 0
"""
client = get_monolith_client()
kwargs = {'app-id': app_id}
if region:
kwargs['region'] = region.slug
today = datetime.datetime.today()
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
try:
count_1 = sum(
c['count'] for c in
client('app_installs', days_ago(7), today, 'week', **kwargs)
if c.get('count'))
except ValueError as e:
task_log.info('Call to ES failed: {0}'.format(e))
count_1 = 0
# If count_1 isn't more than 100, stop here to avoid extra Monolith calls.
if not count_1 > 100:
return 0.0
# Get the average installs for the prior 3 weeks. Don't use the `len` of
# the returned counts because of week boundaries.
try:
count_3 = sum(
c['count'] for c in
client('app_installs', days_ago(28), days_ago(8), 'week', **kwargs)
if c.get('count')) / 3
except ValueError as e:
task_log.info('Call to ES failed: {0}'.format(e))
count_3 = 0
if count_3 > 1:
return (count_1 - count_3) / count_3
else:
return 0.0
def update_downloads(ids, **kw):
client = get_monolith_client()
today = datetime.date.today()
count = 0
for app in Webapp.objects.filter(id__in=ids).no_transforms():
kwargs = {'app-id': app.id}
# Get weekly downloads.
#
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
try:
weekly = sum(
c['count'] for c in
client('app_installs', days_ago(7).date(), today, 'week',
**kwargs)
if c.get('count'))
except ValueError as e:
task_log.info('Call to ES failed: {0}'.format(e))
weekly = 0
# Get total downloads.
#
# The monolith client lib doesn't handle this for us so we send a raw
# ES query to Monolith.
query = {'query': {'match_all': {}},
'facets': {
'installs': {
'statistical': {'field': 'app_installs'},
'facet_filter': {'term': kwargs}}},
'size': 0}
try:
resp = client.raw(query)
total = resp.get('facets', {}).get('installs', {}).get('total', 0)
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
total = 0
# Update Webapp object, if needed.
update = False
signal = False
if weekly != app.weekly_downloads:
update = True
signal = True
if total != app.total_downloads:
update = True
if update:
# Note: Calling `update` will trigger a reindex on the app if
# `_signal` is True. Since we only index `weekly_downloads`, we
# can skip reindexing if this hasn't changed.
count += 1
app.update(weekly_downloads=weekly, total_downloads=total,
_signal=signal)
task_log.info('App downloads updated for %s out of %s apps.'
% (count, len(ids)))
def _app_downloads(app):
stats = {"last_7_days": 0, "last_24_hours": 0, "alltime": 0}
if app.is_webapp():
# Webapps populate these fields via Monolith.
stats["last_24_hours"] = "N/A"
stats["last_7_days"] = app.weekly_downloads
stats["alltime"] = app.total_downloads
return stats
qs = DownloadCount.objects.filter(addon=app)
def sum_(qs):
return qs.aggregate(total=Sum("count"))["total"] or 0
stats["last_24_hours"] = sum_(qs.filter(date__gt=days_ago(1).date()))
stats["last_7_days"] = sum_(qs.filter(date__gte=days_ago(7).date()))
stats["alltime"] = sum_(qs)
return stats
def _app_downloads(app):
stats = {'last_7_days': 0, 'last_24_hours': 0, 'alltime': 0}
if app.is_webapp():
# Webapps populate these fields via Monolith.
stats['last_24_hours'] = 'N/A'
stats['last_7_days'] = app.weekly_downloads
stats['alltime'] = app.total_downloads
return stats
qs = DownloadCount.objects.filter(addon=app)
def sum_(qs):
return qs.aggregate(total=Sum('count'))['total'] or 0
stats['last_24_hours'] = sum_(qs.filter(date__gt=days_ago(1).date()))
stats['last_7_days'] = sum_(qs.filter(date__gte=days_ago(7).date()))
stats['alltime'] = sum_(qs)
return stats
def _app_downloads(app):
stats = {'last_7_days': 0,
'last_24_hours': 0,
'alltime': 0}
if app.is_webapp():
# Webapps populate these fields via Monolith.
stats['last_24_hours'] = 'N/A'
stats['last_7_days'] = app.weekly_downloads
stats['alltime'] = app.total_downloads
return stats
qs = DownloadCount.objects.filter(addon=app)
def sum_(qs):
return qs.aggregate(total=Sum('count'))['total'] or 0
stats['last_24_hours'] = sum_(qs.filter(date__gt=days_ago(1).date()))
stats['last_7_days'] = sum_(qs.filter(date__gte=days_ago(7).date()))
stats['alltime'] = sum_(qs)
return stats
def _weekly_theme_counts():
"""Returns unreviewed themes progress."""
base_filters = {
'pending_themes': Addon.objects.filter(
type=amo.ADDON_PERSONA, status=amo.STATUS_PENDING),
'flagged_themes': Addon.objects.filter(
type=amo.ADDON_PERSONA, status=amo.STATUS_REVIEW_PENDING),
'rereview_themes': RereviewQueueTheme.objects.all(),
}
theme_counts = {}
for queue_type, qs in base_filters.iteritems():
theme_counts[queue_type] = {
'week': qs.filter(created__gte=days_ago(7)).count()
}
return theme_counts
def _weekly_theme_counts():
"""Returns unreviewed themes progress."""
base_filters = {
'pending_themes': Addon.objects.filter(
type=amo.ADDON_PERSONA, status=amo.STATUS_PENDING),
'flagged_themes': Addon.objects.filter(
type=amo.ADDON_PERSONA, status=amo.STATUS_REVIEW_PENDING),
'rereview_themes': RereviewQueueTheme.objects.all(),
}
theme_counts = {}
for queue_type, qs in base_filters.iteritems():
theme_counts[queue_type] = {
'week': qs.filter(created__gte=days_ago(7)).count()
}
return theme_counts
def update_downloads(ids, **kw):
client = get_monolith_client()
count = 0
for app in Webapp.objects.filter(id__in=ids).no_transforms():
appid = {'app-id': app.id}
# Get weekly downloads.
query = {
'query': {
'match_all': {}
},
'facets': {
'installs': {
'date_histogram': {
'value_field': 'app_installs',
'interval': 'week',
'key_field': 'date',
},
'facet_filter': {
'and':
[{
'term': appid
},
{
'range': {
'date': {
'gte':
days_ago(8).date().strftime('%Y-%m-%d'),
'lte':
days_ago(1).date().strftime('%Y-%m-%d'),
}
}
}]
}
}
},
'size': 0
}
try:
resp = client.raw(query)
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
weekly = sum(c['total'] for c in resp.get('facets', {}).get(
'installs', {}).get('entries') if c.get('total'))
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
weekly = 0
# Get total downloads.
query = {
'query': {
'match_all': {}
},
'facets': {
'installs': {
'statistical': {
'field': 'app_installs'
},
'facet_filter': {
'term': appid
}
}
},
'size': 0
}
try:
resp = client.raw(query)
total = resp.get('facets', {}).get('installs', {}).get('total', 0)
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
total = 0
# Update Webapp object, if needed.
update = False
signal = False
if weekly != app.weekly_downloads:
update = True
signal = True
if total != app.total_downloads:
update = True
if update:
# Note: Calling `update` will trigger a reindex on the app if
# `_signal` is True. Since we only index `weekly_downloads`, we
# can skip reindexing if this hasn't changed.
count += 1
app.update(weekly_downloads=weekly,
total_downloads=total,
_signal=signal)
task_log.info('App downloads updated for %s out of %s apps.' %
(count, len(ids)))
def update_downloads(ids, **kw):
client = get_monolith_client()
count = 0
for app in Webapp.objects.filter(id__in=ids).no_transforms():
appid = {'app-id': app.id}
# Get weekly downloads.
query = {
'query': {'match_all': {}},
'facets': {
'installs': {
'date_histogram': {
'value_field': 'app_installs',
'interval': 'week',
'key_field': 'date',
},
'facet_filter': {
'and': [
{'term': appid},
{'range': {'date': {
'gte': days_ago(8).date().strftime('%Y-%m-%d'),
'lte': days_ago(1).date().strftime('%Y-%m-%d'),
}}}
]
}
}
},
'size': 0}
try:
resp = client.raw(query)
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
weekly = sum(
c['total'] for c in
resp.get('facets', {}).get('installs', {}).get('entries')
if c.get('total'))
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
weekly = 0
# Get total downloads.
query = {'query': {'match_all': {}},
'facets': {
'installs': {
'statistical': {'field': 'app_installs'},
'facet_filter': {'term': appid}}},
'size': 0}
try:
resp = client.raw(query)
total = resp.get('facets', {}).get('installs', {}).get('total', 0)
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
total = 0
# Update Webapp object, if needed.
update = False
signal = False
if weekly != app.weekly_downloads:
update = True
signal = True
if total != app.total_downloads:
update = True
if update:
# Note: Calling `update` will trigger a reindex on the app if
# `_signal` is True. Since we only index `weekly_downloads`, we
# can skip reindexing if this hasn't changed.
count += 1
app.update(weekly_downloads=weekly, total_downloads=total,
_signal=signal)
task_log.info('App downloads updated for %s out of %s apps.'
% (count, len(ids)))
def update_downloads(ids, **kw):
client = get_monolith_client()
today = datetime.date.today()
count = 0
for app in Webapp.objects.filter(id__in=ids).no_transforms():
kwargs = {'app-id': app.id}
# Get weekly downloads.
#
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
try:
weekly = sum(
c['count']
for c in client('app_installs',
days_ago(7).date(), today, 'week', **kwargs)
if c.get('count'))
except ValueError as e:
task_log.info('Call to ES failed: {0}'.format(e))
weekly = 0
# Get total downloads.
#
# The monolith client lib doesn't handle this for us so we send a raw
# ES query to Monolith.
query = {
'query': {
'match_all': {}
},
'facets': {
'installs': {
'statistical': {
'field': 'app_installs'
},
'facet_filter': {
'term': kwargs
}
}
},
'size': 0
}
try:
resp = client.raw(query)
total = resp.get('facets', {}).get('installs', {}).get('total', 0)
except Exception as e:
task_log.info('Call to ES failed: {0}'.format(e))
total = 0
# Update Webapp object, if needed.
update = False
signal = False
if weekly != app.weekly_downloads:
update = True
signal = True
if total != app.total_downloads:
update = True
if update:
# Note: Calling `update` will trigger a reindex on the app if
# `_signal` is True. Since we only index `weekly_downloads`, we
# can skip reindexing if this hasn't changed.
count += 1
app.update(weekly_downloads=weekly,
total_downloads=total,
_signal=signal)
task_log.info('App downloads updated for %s out of %s apps.' %
(count, len(ids)))
def update_downloads(ids, **kw):
client = get_monolith_client()
count = 0
for app in Webapp.objects.filter(id__in=ids).no_transforms():
appid = {"app-id": app.id}
# Get weekly downloads.
query = {
"query": {"match_all": {}},
"facets": {
"installs": {
"date_histogram": {"value_field": "app_installs", "interval": "week", "key_field": "date"},
"facet_filter": {
"and": [
{"term": appid},
{
"range": {
"date": {
"gte": days_ago(8).date().strftime("%Y-%m-%d"),
"lte": days_ago(1).date().strftime("%Y-%m-%d"),
}
}
},
]
},
}
},
"size": 0,
}
try:
resp = client.raw(query)
# If we query monolith with interval=week and the past 7 days
# crosses a Monday, Monolith splits the counts into two. We want
# the sum over the past week so we need to `sum` these.
weekly = sum(
c["total"] for c in resp.get("facets", {}).get("installs", {}).get("entries") if c.get("total")
)
except Exception as e:
task_log.info("Call to ES failed: {0}".format(e))
weekly = 0
# Get total downloads.
query = {
"query": {"match_all": {}},
"facets": {"installs": {"statistical": {"field": "app_installs"}, "facet_filter": {"term": appid}}},
"size": 0,
}
try:
resp = client.raw(query)
total = resp.get("facets", {}).get("installs", {}).get("total", 0)
except Exception as e:
task_log.info("Call to ES failed: {0}".format(e))
total = 0
# Update Webapp object, if needed.
update = False
signal = False
if weekly != app.weekly_downloads:
update = True
signal = True
if total != app.total_downloads:
update = True
if update:
# Note: Calling `update` will trigger a reindex on the app if
# `_signal` is True. Since we only index `weekly_downloads`, we
# can skip reindexing if this hasn't changed.
count += 1
app.update(weekly_downloads=weekly, total_downloads=total, _signal=signal)
task_log.info("App downloads updated for %s out of %s apps." % (count, len(ids)))