def index_finance_total_inapp(addons, **kw):
"""
Bug 758071
Aggregates financial stats from all of the contributions for in-apps.
"""
es = elasticutils.get_es()
log.info('Indexing total financial in-app stats for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
try:
key = ord_word('totinapp' + str(addon) + inapp_name)
data = search.get_finance_total_inapp(qs, addon, inapp_name)
if not already_indexed(InappPayment, data):
InappPayment.index(data, bulk=True, id=key)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_inapp.retry(args=[addons], exc=exc)
raise
def index_finance_total_inapp(addons, **kw):
"""
Bug 758071
Aggregates financial stats from all of the contributions for in-apps.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
try:
key = ord_word('totinapp' + str(addon) + inapp_name)
data = search.get_finance_total_inapp(qs, addon, inapp_name)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_inapp.retry(args=[addons], exc=exc, **kw)
raise
def index_finance_daily_inapp(ids, **kw):
"""
Similar to index_finance_daily, except for InappPayments.
ids -- ids of mkt.stats.webapps.InappPayment objects
"""
es = elasticutils.get_es()
# Get contributions.
qs = (InappPayment.objects.filter(id__in=ids).order_by('created').values(
'name', 'config__addon', 'created'))
log.info('[%s] Indexing %s in-app payments for daily stats.' %
(qs[0]['created'], len(ids)))
# It's defaultdicts all the way down.
addons_inapps_dates = defaultdict(
lambda: defaultdict(lambda: defaultdict(int)))
for payment in qs:
addon = payment['config__addon']
inapp = payment['name']
date = payment['created'].strftime('%Y%m%d')
# Date for add-on not processed, index it and give it key.
if not date in addons_inapps_dates[addon][inapp]:
key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date)))
data = search.get_finance_daily_inapp(payment)
try:
if not already_indexed(InappPayment, data):
InappPayment.index(data, bulk=True, id=key)
addons_inapps_dates[addon][inapp][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_daily_inapp.retry(args=[ids], exc=exc)
raise
def index_finance_total_inapp_by_src(addons, **kw):
"""
Total finance in-app stats, src breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by src for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct sources for given in-app.
sources = set(qs.values_list('contribution__source', flat=True))
for source in sources:
try:
key = ord_word('srcinapp' + str(addon) + inapp_name +
source.lower())
try:
data = search.get_finance_total_inapp(qs,
addon,
inapp_name,
'source',
source=source)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data,
bulk=True,
id=key,
index=index)
except Exception, e:
# We ignore this error for now. See #805181
pass
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_src.retry(args=[addons],
exc=exc,
**kw)
raise
def index_finance_total_inapp_by_currency(addons, **kw):
"""
Bug 758071
Total finance in-app stats, currency breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by currency for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(
InappPayment.objects.filter(config__addon=addon).values_list(
'name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct currencies for given in-app.
currencies = set(
qs.values_list('contribution__currency', flat=True))
for currency in currencies:
try:
key = ord_word('curinapp' + str(addon) + inapp_name +
currency.lower())
data = search.get_finance_total_inapp(qs,
addon,
inapp_name,
'currency',
currency=currency)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data,
bulk=True,
id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_currency.retry(args=[addons],
exc=exc,
**kw)
raise
def index_finance_total_inapp_by_src(addons, **kw):
"""
Total finance in-app stats, src breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by src for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct sources for given in-app.
sources = set(qs.values_list('contribution__source',
flat=True))
for source in sources:
try:
key = ord_word('srcinapp' + str(addon) + inapp_name +
source.lower())
try:
data = search.get_finance_total_inapp(
qs, addon, inapp_name, 'source', source=source)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
except Exception, e:
# We ignore this error for now. See #805181
pass
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_src.retry(args=[addons],
exc=exc, **kw)
raise
def index_finance_total_inapp_by_currency(addons, **kw):
"""
Bug 758071
Total finance in-app stats, currency breakdown.
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
log.info('Indexing total financial in-app stats by currency for %s apps.' %
len(addons))
for addon in addons:
# Get all in-app names for given addon.
inapps = set(InappPayment.objects.filter(config__addon=addon).
values_list('name', flat=True))
for inapp_name in inapps:
# Get all in-app payments for given in-app.
qs = InappPayment.objects.filter(name=inapp_name,
contribution__uuid=None)
if not qs.exists():
continue
# Get a list of distinct currencies for given in-app.
currencies = set(qs.values_list('contribution__currency',
flat=True))
for currency in currencies:
try:
key = ord_word('curinapp' + str(addon) + inapp_name +
currency.lower())
data = search.get_finance_total_inapp(
qs, addon, inapp_name, 'currency', currency=currency)
for index in get_indices(index):
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key,
index=index)
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_currency.retry(args=[addons],
exc=exc, **kw)
raise
def index_finance_daily_inapp(ids, **kw):
"""
Similar to index_finance_daily, except for InappPayments.
ids -- ids of mkt.stats.webapps.InappPayment objects
"""
index = kw.get('index', InappPayment._get_index())
es = amo.search.get_es()
# Get contributions.
qs = (InappPayment.objects.filter(id__in=ids)
.order_by('created').values('name',
'config__addon',
'created'))
log.info('[%s] Indexing %s in-app payments for daily stats.' %
(qs[0]['created'], len(ids)))
# It's defaultdicts all the way down.
addons_inapps_dates = defaultdict(lambda: defaultdict(
lambda: defaultdict(int)))
for payment in qs:
addon = payment['config__addon']
inapp = payment['name']
date = payment['created'].strftime('%Y%m%d')
# Date for add-on not processed, index it and give it key.
if not date in addons_inapps_dates[addon][inapp]:
key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date)))
data = search.get_finance_daily_inapp(payment)
try:
if not already_indexed(InappPayment, data, index):
InappPayment.index(data, bulk=True, id=key, index=index)
addons_inapps_dates[addon][inapp][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_daily_inapp.retry(args=[ids], exc=exc, **kw)
raise
