def _progress():
"""Returns unreviewed apps progress.
Return the number of apps still unreviewed for a given period of time and
the percentage.
"""
queues_helper = ReviewersQueuesHelper()
base_filters = {
'pending': (queues_helper.get_pending_queue(),
'nomination'),
'homescreen': (queues_helper.get_homescreen_queue(),
'nomination'),
'rereview': (queues_helper.get_rereview_queue(),
'created'),
'escalated': (queues_helper.get_escalated_queue(),
'created'),
'updates': (queues_helper.get_updates_queue(),
'nomination')
}
operators_and_values = {
'new': ('gt', days_ago(5)),
'med': ('range', (days_ago(10), days_ago(5))),
'old': ('lt', days_ago(10)),
'week': ('gte', days_ago(7))
}
types = base_filters.keys()
progress = {}
for t in types:
tmp = {}
base_query, field = base_filters[t]
for k in operators_and_values.keys():
operator, value = operators_and_values[k]
filter_ = {}
filter_['%s__%s' % (field, operator)] = value
tmp[k] = base_query.filter(**filter_).count()
progress[t] = tmp
def pct(p, t):
# Return the percent of (p)rogress out of (t)otal.
return (p / float(t)) * 100 if p > 0 else 0
percentage = {}
for t in types:
total = progress[t]['new'] + progress[t]['med'] + progress[t]['old']
percentage[t] = {}
for duration in ('new', 'med', 'old'):
percentage[t][duration] = pct(progress[t][duration], total)
return (progress, percentage)
def _progress():
"""Returns unreviewed apps progress.
Return the number of apps still unreviewed for a given period of time and
the percentage.
"""
queues_helper = ReviewersQueuesHelper()
base_filters = {
'pending': (queues_helper.get_pending_queue(),
'nomination'),
'rereview': (queues_helper.get_rereview_queue(),
'created'),
'escalated': (queues_helper.get_escalated_queue(),
'created'),
'updates': (queues_helper.get_updates_queue(),
'nomination')
}
operators_and_values = {
'new': ('gt', days_ago(5)),
'med': ('range', (days_ago(10), days_ago(5))),
'old': ('lt', days_ago(10)),
'week': ('gte', days_ago(7))
}
types = base_filters.keys()
progress = {}
for t in types:
tmp = {}
base_query, field = base_filters[t]
for k in operators_and_values.keys():
operator, value = operators_and_values[k]
filter_ = {}
filter_['%s__%s' % (field, operator)] = value
tmp[k] = base_query.filter(**filter_).count()
progress[t] = tmp
def pct(p, t):
# Return the percent of (p)rogress out of (t)otal.
return (p / float(t)) * 100 if p > 0 else 0
percentage = {}
for t in types:
total = progress[t]['new'] + progress[t]['med'] + progress[t]['old']
percentage[t] = {}
for duration in ('new', 'med', 'old'):
percentage[t][duration] = pct(progress[t][duration], total)
return (progress, percentage)
def mkt_gc(**kw):
"""Site-wide garbage collections."""
log.info('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90)).exclude(
action__in=mkt.LOG_KEEP).values_list('id', flat=True))
for chunk in chunked(logs, 100):
chunk.sort()
log.info('Deleting log entries: %s' % str(chunk))
delete_logs.delay(chunk)
# Clear oauth nonce rows. These expire after 10 minutes but we're just
# clearing those that are more than 1 day old.
Nonce.objects.filter(created__lt=days_ago(1)).delete()
# Delete the dump apps over 30 days.
_remove_stale_files(os.path.join(settings.DUMPED_APPS_PATH, 'tarballs'),
settings.DUMPED_APPS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=public_storage)
# Delete the dumped user installs over 30 days. Those are using private
# storage.
_remove_stale_files(os.path.join(settings.DUMPED_USERS_PATH, 'tarballs'),
settings.DUMPED_USERS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=private_storage)
# Delete old files in select directories under TMP_PATH.
_remove_stale_files(os.path.join(settings.TMP_PATH, 'preview'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
_remove_stale_files(os.path.join(settings.TMP_PATH, 'icon'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
# Delete stale FileUploads.
for fu in FileUpload.objects.filter(created__lte=days_ago(90)):
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'.format(
uuid=fu.uuid, path=fu.path))
if fu.path:
try:
private_storage.delete(fu.path)
except OSError:
pass
fu.delete()
def mkt_gc(**kw):
"""Site-wide garbage collections."""
log.info('Collecting data to delete')
logs = (ActivityLog.objects.filter(created__lt=days_ago(90))
.exclude(action__in=mkt.LOG_KEEP).values_list('id', flat=True))
for chunk in chunked(logs, 100):
chunk.sort()
log.info('Deleting log entries: %s' % str(chunk))
delete_logs.delay(chunk)
# Clear oauth nonce rows. These expire after 10 minutes but we're just
# clearing those that are more than 1 day old.
Nonce.objects.filter(created__lt=days_ago(1)).delete()
# Delete the dump apps over 30 days.
_remove_stale_files(os.path.join(settings.DUMPED_APPS_PATH, 'tarballs'),
settings.DUMPED_APPS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=public_storage)
# Delete the dumped user installs over 30 days. Those are using private
# storage.
_remove_stale_files(os.path.join(settings.DUMPED_USERS_PATH, 'tarballs'),
settings.DUMPED_USERS_DAYS_DELETE,
'Deleting old tarball: {0}',
storage=private_storage)
# Delete old files in select directories under TMP_PATH.
_remove_stale_files(os.path.join(settings.TMP_PATH, 'preview'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
_remove_stale_files(os.path.join(settings.TMP_PATH, 'icon'),
settings.TMP_PATH_DAYS_DELETE,
'Deleting TMP_PATH file: {0}',
storage=private_storage)
# Delete stale FileUploads.
for fu in FileUpload.objects.filter(created__lte=days_ago(90)):
log.debug(u'[FileUpload:{uuid}] Removing file: {path}'
.format(uuid=fu.uuid, path=fu.path))
if fu.path:
try:
private_storage.delete(fu.path)
except OSError:
pass
fu.delete()
def _get_trending(app_id):
"""
Calculate trending for app for all regions and per region.
a = installs from 8 days ago to 1 day ago
b = installs from 29 days ago to 9 days ago, averaged per week
trending = (a - b) / b if a > 100 and b > 1 else 0
Returns value in the format of::
{'all': <global trending score>,
<region_slug>: <regional trending score>,
...}
"""
# How many app installs are required in the prior week to be considered
# "trending". Adjust this as total Marketplace app installs increases.
#
# Note: AMO uses 1000.0 for add-ons.
PRIOR_WEEK_INSTALL_THRESHOLD = 100.0
client = get_monolith_client()
week1 = {
'filter': {
'range': {
'date': {
'gte': days_ago(8).date().isoformat(),
'lte': days_ago(1).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
week3 = {
'filter': {
'range': {
'date': {
'gte': days_ago(29).date().isoformat(),
'lte': days_ago(9).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
query = {
'query': {
'filtered': {
'query': {'match_all': {}},
'filter': {'term': {'app-id': app_id}}
}
},
'aggregations': {
'week1': week1,
'week3': week3,
'region': {
'terms': {
'field': 'region',
# Add size so we get all regions, not just the top 10.
'size': len(mkt.regions.ALL_REGIONS)
},
'aggregations': {
'week1': week1,
'week3': week3
}
}
},
'size': 0
}
try:
res = client.raw(query)
except ValueError as e:
task_log.error('Error response from Monolith: {0}'.format(e))
return {}
if 'aggregations' not in res:
task_log.error('No installs for app {}'.format(app_id))
return {}
def _score(week1, week3):
# If last week app installs are < 100, this app isn't trending.
if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
return 0.0
score = 0.0
if week3 > 1.0:
score = (week1 - week3) / week3
if score < 0.0:
score = 0.0
return score
# Global trending score.
week1 = res['aggregations']['week1']['total_installs']['value']
week3 = res['aggregations']['week3']['total_installs']['value'] / 3.0
if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
# If global installs over the last week aren't over 100, we
# short-circuit and return a zero-like value as this is not a trending
# app by definition. Since global installs aren't above 100, per-region
# installs won't be either.
return {}
results = {
'all': _score(week1, week3)
}
if 'region' in res['aggregations']:
for regional_res in res['aggregations']['region']['buckets']:
region_slug = regional_res['key']
week1 = regional_res['week1']['total_installs']['value']
week3 = regional_res['week3']['total_installs']['value'] / 3.0
results[region_slug] = _score(week1, week3)
return results
def _get_installs(app_id):
"""
Calculate popularity of app for all regions and per region.
Returns value in the format of::
{'all': <global installs>,
<region_slug>: <regional installs>,
...}
"""
# How many days back do we include when calculating popularity.
POPULARITY_PERIOD = 90
client = get_monolith_client()
popular = {
'filter': {
'range': {
'date': {
'gte': days_ago(POPULARITY_PERIOD).date().isoformat(),
'lte': days_ago(1).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
query = {
'query': {
'filtered': {
'query': {'match_all': {}},
'filter': {'term': {'app-id': app_id}}
}
},
'aggregations': {
'popular': popular,
'region': {
'terms': {
'field': 'region',
# Add size so we get all regions, not just the top 10.
'size': len(mkt.regions.ALL_REGIONS)
},
'aggregations': {
'popular': popular
}
}
},
'size': 0
}
try:
res = client.raw(query)
except ValueError as e:
task_log.error('Error response from Monolith: {0}'.format(e))
return {}
if 'aggregations' not in res:
task_log.error('No installs for app {}'.format(app_id))
return {}
results = {
'all': res['aggregations']['popular']['total_installs']['value']
}
if 'region' in res['aggregations']:
for regional_res in res['aggregations']['region']['buckets']:
region_slug = regional_res['key']
popular = regional_res['popular']['total_installs']['value']
results[region_slug] = popular
return results
def _get_trending(app_id):
"""
Calculate trending for app for all regions and per region.
a = installs from 8 days ago to 1 day ago
b = installs from 29 days ago to 9 days ago, averaged per week
trending = (a - b) / b if a > 100 and b > 1 else 0
Returns value in the format of::
{'all': <global trending score>,
<region_slug>: <regional trending score>,
...}
"""
# How many app installs are required in the prior week to be considered
# "trending". Adjust this as total Marketplace app installs increases.
#
# Note: AMO uses 1000.0 for add-ons.
PRIOR_WEEK_INSTALL_THRESHOLD = 100.0
client = get_monolith_client()
week1 = {
'filter': {
'range': {
'date': {
'gte': days_ago(8).date().isoformat(),
'lte': days_ago(1).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
week3 = {
'filter': {
'range': {
'date': {
'gte': days_ago(29).date().isoformat(),
'lte': days_ago(9).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
},
'filter': {
'term': {
'app-id': app_id
}
}
}
},
'aggregations': {
'week1': week1,
'week3': week3,
'region': {
'terms': {
'field': 'region',
# Add size so we get all regions, not just the top 10.
'size': len(mkt.regions.ALL_REGIONS)
},
'aggregations': {
'week1': week1,
'week3': week3
}
}
},
'size': 0
}
try:
res = client.raw(query)
except ValueError as e:
task_log.error('Error response from Monolith: {0}'.format(e))
return {}
if 'aggregations' not in res:
task_log.error('No installs for app {}'.format(app_id))
return {}
def _score(week1, week3):
# If last week app installs are < 100, this app isn't trending.
if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
return 0.0
score = 0.0
if week3 > 1.0:
score = (week1 - week3) / week3
if score < 0.0:
score = 0.0
return score
# Global trending score.
week1 = res['aggregations']['week1']['total_installs']['value']
week3 = res['aggregations']['week3']['total_installs']['value'] / 3.0
if week1 < PRIOR_WEEK_INSTALL_THRESHOLD:
# If global installs over the last week aren't over 100, we
# short-circuit and return a zero-like value as this is not a trending
# app by definition. Since global installs aren't above 100, per-region
# installs won't be either.
return {}
results = {'all': _score(week1, week3)}
if 'region' in res['aggregations']:
for regional_res in res['aggregations']['region']['buckets']:
region_slug = regional_res['key']
week1 = regional_res['week1']['total_installs']['value']
week3 = regional_res['week3']['total_installs']['value'] / 3.0
results[region_slug] = _score(week1, week3)
return results
def _get_installs(app_id):
"""
Calculate popularity of app for all regions and per region.
Returns value in the format of::
{'all': <global installs>,
<region_slug>: <regional installs>,
...}
"""
# How many days back do we include when calculating popularity.
POPULARITY_PERIOD = 90
client = get_monolith_client()
popular = {
'filter': {
'range': {
'date': {
'gte': days_ago(POPULARITY_PERIOD).date().isoformat(),
'lte': days_ago(1).date().isoformat()
}
}
},
'aggs': {
'total_installs': {
'sum': {
'field': 'app_installs'
}
}
}
}
query = {
'query': {
'filtered': {
'query': {
'match_all': {}
},
'filter': {
'term': {
'app-id': app_id
}
}
}
},
'aggregations': {
'popular': popular,
'region': {
'terms': {
'field': 'region',
# Add size so we get all regions, not just the top 10.
'size': len(mkt.regions.ALL_REGIONS)
},
'aggregations': {
'popular': popular
}
}
},
'size': 0
}
try:
res = client.raw(query)
except ValueError as e:
task_log.error('Error response from Monolith: {0}'.format(e))
return {}
if 'aggregations' not in res:
task_log.error('No installs for app {}'.format(app_id))
return {}
results = {
'all': res['aggregations']['popular']['total_installs']['value']
}
if 'region' in res['aggregations']:
for regional_res in res['aggregations']['region']['buckets']:
region_slug = regional_res['key']
popular = regional_res['popular']['total_installs']['value']
results[region_slug] = popular
return results