def index_finance_daily_inapp(ids, **kw):
"""
Similar to index_finance_daily, except for InappPayments.
ids -- ids of mkt.stats.webapps.InappPayment objects
"""
es = elasticutils.get_es()
# Get contributions.
qs = (InappPayment.objects.filter(id__in=ids).order_by('created').values(
'name', 'config__addon', 'created'))
log.info('[%s] Indexing %s in-app payments for daily stats.' %
(qs[0]['created'], len(ids)))
# It's defaultdicts all the way down.
addons_inapps_dates = defaultdict(
lambda: defaultdict(lambda: defaultdict(int)))
for payment in qs:
addon = payment['config__addon']
inapp = payment['name']
date = payment['created'].strftime('%Y%m%d')
# Date for add-on not processed, index it and give it key.
if not date in addons_inapps_dates[addon][inapp]:
key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date)))
data = search.get_finance_daily_inapp(payment)
try:
if not already_indexed(InappPayment, data):
InappPayment.index(data, bulk=True, id=key)
addons_inapps_dates[addon][inapp][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_daily_inapp.retry(args=[ids], exc=exc)
raise
def index_finance_total_inapp(addons, **kw):
"""
Bug 758071
Aggregates financial stats from all of the contributions for in-apps.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
try:
key = ord_word('totinapp' + str(addon) + inapp_name)
data = search.get_finance_total_inapp(qs, addon, inapp_name)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_inapp.retry(args=[addons], exc=exc, **kw)
raise
def index_finance_total_inapp(addons, **kw):
"""
Bug 758071
Aggregates financial stats from all of the contributions for in-apps.
"""
es = elasticutils.get_es()
log.info('Indexing total financial in-app stats for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
try:
key = ord_word('totinapp' + str(addon) + inapp_name)
data = search.get_finance_total_inapp(qs, addon, inapp_name)
if not already_indexed(InappPayment, data):
InappPayment.index(data, bulk=True, id=key)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_inapp.retry(args=[addons], exc=exc)
raise
def index_finance_total_inapp_by_src(addons, **kw):
"""
Total finance in-app stats, src breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by src for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct sources for given in-app.
sources = set(qs.values_list('contribution__source', flat=True))
for source in sources:
try:
key = ord_word('srcinapp' + str(addon) + inapp_name +
source.lower())
try:
data = search.get_finance_total_inapp(qs,
addon,
inapp_name,
'source',
source=source)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data,
bulk=True,
id=key,
index=index)
except Exception, e:
# We ignore this error for now. See #805181
pass
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_src.retry(args=[addons],
exc=exc,
**kw)
raise
def test_index(self):
tasks.index_finance_daily_inapp.delay(self.ids)
self.refresh(timesleep=1)
document = (
InappPayment.search()
.filter(config__addon=self.app.pk)
.values_dict("date", "inapp", "revenue", "count", "refunds")[0]
)
eq_(self.inapp_name, document["inapp"])
date = document["date"]
ex_date = self.expected[self.inapp_name]["date"]
eq_((date.year, date.month, date.day), (ex_date.year, ex_date.month, ex_date.day))
document = {
self.inapp_name: {
"count": document["count"],
"revenue": int(document["revenue"]),
"refunds": document["refunds"],
}
}
del (self.expected[self.inapp_name]["date"])
self.expected[self.inapp_name]["revenue"] = int(self.expected[self.inapp_name]["revenue"])
eq_(document, self.expected)
def test_index(self):
tasks.index_finance_daily_inapp.delay(self.ids)
self.refresh(timesleep=1)
document = InappPayment.search().filter(
config__addon=self.app.pk).values_dict('date', 'inapp', 'revenue',
'count', 'refunds')[0]
eq_(self.inapp_name, document['inapp'])
date = document['date']
ex_date = self.expected[self.inapp_name]['date']
eq_((date.year, date.month, date.day),
(ex_date.year, ex_date.month, ex_date.day))
document = {
self.inapp_name: {
'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']
}
}
del (self.expected[self.inapp_name]['date'])
self.expected[self.inapp_name]['revenue'] = (int(
self.expected[self.inapp_name]['revenue']))
eq_(document, self.expected)
def test_index(self):
tasks.index_finance_total_inapp_by_currency([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for currency in self.currencies:
# For some reason, query fails if uppercase letter in filter.
document = (InappPayment.search().
filter(addon=self.app.pk, inapp=self.inapp_name,
currency=currency.lower()).
values_dict('currency', 'revenue', 'count',
'refunds',
'revenue_non_normalized'))[0]
document = {'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds'],
'revenue_non_normalized':
int(document['revenue_non_normalized'])}
expected_inapp = self.expected[self.inapp_name]
expected_inapp[currency]['revenue'] = (
int(expected_inapp[currency]['revenue'])
)
expected_inapp[currency]['revenue_non_normalized'] = (
int(expected_inapp[currency]['revenue_non_normalized'])
)
eq_(document, expected_inapp[currency])
def test_index(self):
tasks.index_finance_total_inapp_by_currency([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for currency in self.currencies:
# For some reason, query fails if uppercase letter in filter.
document = (InappPayment.search().filter(
addon=self.app.pk,
inapp=self.inapp_name,
currency=currency.lower()).values_dict(
'currency', 'revenue', 'count', 'refunds',
'revenue_non_normalized'))[0]
document = {
'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds'],
'revenue_non_normalized':
int(document['revenue_non_normalized'])
}
expected_inapp = self.expected[self.inapp_name]
expected_inapp[currency]['revenue'] = (int(
expected_inapp[currency]['revenue']))
expected_inapp[currency]['revenue_non_normalized'] = (int(
expected_inapp[currency]['revenue_non_normalized']))
eq_(document, expected_inapp[currency])
def test_index(self):
tasks.index_finance_daily_inapp.delay(self.ids)
self.refresh(timesleep=1)
document = InappPayment.search().filter(config__addon=self.app.pk
).values_dict('date', 'inapp', 'revenue', 'count', 'refunds')[0]
eq_(self.inapp_name, document['inapp'])
date = document['date']
ex_date = self.expected[self.inapp_name]['date']
eq_((date.year, date.month, date.day),
(ex_date.year, ex_date.month, ex_date.day))
document = {
self.inapp_name: {
'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']
}
}
del(self.expected[self.inapp_name]['date'])
self.expected[self.inapp_name]['revenue'] = (
int(self.expected[self.inapp_name]['revenue']))
eq_(document, self.expected)
def index_finance_total_inapp_by_currency(addons, **kw):
"""
Bug 758071
Total finance in-app stats, currency breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by currency for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct currencies for given in-app.
currencies = set(
qs.values_list('contribution__currency', flat=True))
for currency in currencies:
try:
key = ord_word('curinapp' + str(addon) + inapp_name +
currency.lower())
data = search.get_finance_total_inapp(qs,
addon,
inapp_name,
'currency',
currency=currency)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data,
bulk=True,
id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_currency.retry(args=[addons],
exc=exc,
**kw)
raise
def index_finance_total_inapp_by_src(addons, **kw):
"""
Total finance in-app stats, src breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by src for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct sources for given in-app.
sources = set(qs.values_list('contribution__source',
flat=True))
for source in sources:
try:
key = ord_word('srcinapp' + str(addon) + inapp_name +
source.lower())
try:
data = search.get_finance_total_inapp(
qs, addon, inapp_name, 'source', source=source)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
except Exception, e:
# We ignore this error for now. See #805181
pass
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_src.retry(args=[addons],
exc=exc, **kw)
raise
def index_finance_total_inapp_by_currency(addons, **kw):
"""
Bug 758071
Total finance in-app stats, currency breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by currency for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct currencies for given in-app.
currencies = set(qs.values_list('contribution__currency',
flat=True))
for currency in currencies:
try:
key = ord_word('curinapp' + str(addon) + inapp_name +
currency.lower())
data = search.get_finance_total_inapp(
qs, addon, inapp_name, 'currency', currency=currency)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_currency.retry(args=[addons],
exc=exc, **kw)
raise
def test_index(self):
tasks.index_finance_total_inapp([self.app.pk])
self.refresh(timesleep=1)
document = (
InappPayment.search()
.filter(addon=self.app.pk, inapp=self.inapp_name)
.values_dict("revenue", "count", "refunds")
)[0]
document = {"count": document["count"], "revenue": int(document["revenue"]), "refunds": document["refunds"]}
self.expected[self.inapp_name]["revenue"] = int(self.expected[self.inapp_name]["revenue"])
eq_(document, self.expected[self.inapp_name])
def index_finance_daily_inapp(ids, **kw):
"""
Similar to index_finance_daily, except for InappPayments.
ids -- ids of mkt.stats.webapps.InappPayment objects
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
# Get contributions.
qs = (InappPayment.objects.filter(id__in=ids)
.order_by('created').values('name',
'config__addon',
'created'))
log.info('[%s] Indexing %s in-app payments for daily stats.' %
(qs[0]['created'], len(ids)))
# It's defaultdicts all the way down.
addons_inapps_dates = defaultdict(lambda: defaultdict(
lambda: defaultdict(int)))
for payment in qs:
addon = payment['config__addon']
inapp = payment['name']
date = payment['created'].strftime('%Y%m%d')
# Date for add-on not processed, index it and give it key.
if not date in addons_inapps_dates[addon][inapp]:
key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date)))
data = search.get_finance_daily_inapp(payment)
try:
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key, index=index)
addons_inapps_dates[addon][inapp][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_daily_inapp.retry(args=[ids], exc=exc, **kw)
raise
def test_index(self):
tasks.index_finance_total_inapp([self.app.pk])
self.refresh(timesleep=1)
document = (InappPayment.search().
filter(addon=self.app.pk, inapp=self.inapp_name).
values_dict('revenue', 'count', 'refunds'))[0]
document = {'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']}
self.expected[self.inapp_name]['revenue'] = int(
self.expected[self.inapp_name]['revenue'])
eq_(document, self.expected[self.inapp_name])
def test_index(self):
tasks.index_finance_total_inapp_by_src([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for source in self.sources:
# For some reason, query fails if uppercase letter in filter.
document = (
InappPayment.search()
.filter(addon=self.app.pk, inapp=self.inapp_name, source=source.lower())
.values_dict("source", "revenue", "count", "refunds")
)[0]
document = {"count": document["count"], "revenue": int(document["revenue"]), "refunds": document["refunds"]}
self.expected_inapp[source]["revenue"] = int(self.expected_inapp[source]["revenue"])
eq_(document, self.expected_inapp[source])
def stats_report(request, addon, report, inapp=None, category_field=None):
"""
Stats page. Passes in context variables into template which is read by the
JS to build a URL. The URL calls a *_series view which determines
necessary arguments for get_series_*. get_series_* queries ES for the data,
which is later formatted into .json or .csv and made available to the JS.
"""
if addon.status is not amo.STATUS_PUBLIC and not check_stats_permission(
request, addon, for_contributions=True, no_raise=True
):
return redirect(addon.get_detail_url())
check_stats_permission(request, addon)
# For inapp, point template to same as non-inapp, but still use
# different report names.
template_name = "appstats/reports/%s.html" % report.replace("_inapp", "")
if inapp:
stats_base_url = addon.get_stats_inapp_url(action="revenue", inapp=inapp)
else:
stats_base_url = reverse("mkt.stats.overview", args=[addon.app_slug])
view = get_report_view(request)
# Get list of in-apps for drop-down in-app selector.
inapps = []
# Until we figure out why ES stores strings in lowercase despite
# the field being set to not analyze, we grab the lowercase version
# from ES and do a case-insensitive query to the ORM to un-lowercase.
inapps_lower = list(
set(payment["inapp"] for payment in list(InappPayment.search().filter(addon=addon.id).values_dict("inapp")))
)
for inapp_name in inapps_lower:
inapps.append(InappPayment.objects.filter(name__iexact=inapp_name)[0].name)
return jingo.render(
request,
template_name,
{
"addon": addon,
"report": report,
"view": view,
"stats_base_url": stats_base_url,
"inapp": inapp,
"inapps": inapps,
},
)
def test_index(self):
tasks.index_finance_total_inapp([self.app.pk])
self.refresh(timesleep=1)
document = (InappPayment.search().filter(
addon=self.app.pk,
inapp=self.inapp_name).values_dict('revenue', 'count',
'refunds'))[0]
document = {
'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']
}
self.expected[self.inapp_name]['revenue'] = int(
self.expected[self.inapp_name]['revenue'])
eq_(document, self.expected[self.inapp_name])
def test_index(self):
tasks.index_finance_total_inapp_by_src([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for source in self.sources:
# For some reason, query fails if uppercase letter in filter.
document = (InappPayment.search().
filter(addon=self.app.pk, inapp=self.inapp_name,
source=source.lower()).
values_dict('source', 'revenue', 'count',
'refunds'))[0]
document = {'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']}
self.expected_inapp[source]['revenue'] = (
int(self.expected_inapp[source]['revenue'])
)
eq_(document, self.expected_inapp[source])
def stats_report(request, addon, report, inapp=None, category_field=None):
"""
Stats page. Passes in context variables into template which is read by the
JS to build a URL. The URL calls a *_series view which determines
necessary arguments for get_series_*. get_series_* queries ES for the data,
which is later formatted into .json or .csv and made available to the JS.
"""
if (addon.status is not amo.STATUS_PUBLIC and not check_stats_permission(
request, addon, for_contributions=True, no_raise=True)):
return redirect(addon.get_detail_url())
check_stats_permission(request, addon)
# For inapp, point template to same as non-inapp, but still use
# different report names.
template_name = 'appstats/reports/%s.html' % report.replace('_inapp', '')
if inapp:
stats_base_url = addon.get_stats_inapp_url(action='revenue',
inapp=inapp)
else:
stats_base_url = reverse('mkt.stats.overview', args=[addon.app_slug])
view = get_report_view(request)
# Get list of in-apps for drop-down in-app selector.
inapps = []
# Until we figure out why ES stores strings in lowercase despite
# the field being set to not analyze, we grab the lowercase version
# from ES and do a case-insensitive query to the ORM to un-lowercase.
inapps_lower = list(
set(payment['inapp'] for payment in list(InappPayment.search().filter(
addon=addon.id).values_dict('inapp'))))
for inapp_name in inapps_lower:
inapps.append(
InappPayment.objects.filter(name__iexact=inapp_name)[0].name)
return jingo.render(
request, template_name, {
'addon': addon,
'report': report,
'view': view,
'stats_base_url': stats_base_url,
'inapp': inapp,
'inapps': inapps,
})
def test_index(self):
tasks.index_finance_total_inapp_by_src([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for source in self.sources:
# For some reason, query fails if uppercase letter in filter.
document = (InappPayment.search().filter(
addon=self.app.pk,
inapp=self.inapp_name,
source=source.lower()).values_dict('source', 'revenue',
'count', 'refunds'))[0]
document = {
'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']
}
self.expected_inapp[source]['revenue'] = (int(
self.expected_inapp[source]['revenue']))
eq_(document, self.expected_inapp[source])
def test_index(self):
tasks.index_finance_total_inapp_by_currency([self.app.pk])
self.refresh(timesleep=1)
# Grab document for each source breakdown and compare.
for currency in self.currencies:
# For some reason, query fails if uppercase letter in filter.
document = (
InappPayment.search()
.filter(addon=self.app.pk, inapp=self.inapp_name, currency=currency.lower())
.values_dict("currency", "revenue", "count", "refunds", "revenue_non_normalized")
)[0]
document = {
"count": document["count"],
"revenue": int(document["revenue"]),
"refunds": document["refunds"],
"revenue_non_normalized": int(document["revenue_non_normalized"]),
}
expected_inapp = self.expected[self.inapp_name]
expected_inapp[currency]["revenue"] = int(expected_inapp[currency]["revenue"])
expected_inapp[currency]["revenue_non_normalized"] = int(expected_inapp[currency]["revenue_non_normalized"])
eq_(document, expected_inapp[currency])
