def index_finance_total_inapp(addons, **kw): """ Bug 758071 Aggregates financial stats from all of the contributions for in-apps. """ es = elasticutils.get_es() log.info('Indexing total financial in-app stats for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set( InappPayment.objects.filter(config__addon=addon).values_list( 'name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue try: key = ord_word('totinapp' + str(addon) + inapp_name) data = search.get_finance_total_inapp(qs, addon, inapp_name) if not already_indexed(InappPayment, data): InappPayment.index(data, bulk=True, id=key) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_inapp.retry(args=[addons], exc=exc) raise
def index_finance_total_inapp(addons, **kw): """ Bug 758071 Aggregates financial stats from all of the contributions for in-apps. """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() log.info('Indexing total financial in-app stats for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set(InappPayment.objects.filter(config__addon=addon). values_list('name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue try: key = ord_word('totinapp' + str(addon) + inapp_name) data = search.get_finance_total_inapp(qs, addon, inapp_name) for index in get_indices(index): if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_inapp.retry(args=[addons], exc=exc, **kw) raise
def index_finance_daily_inapp(ids, **kw): """ Similar to index_finance_daily, except for InappPayments. ids -- ids of mkt.stats.webapps.InappPayment objects """ es = elasticutils.get_es() # Get contributions. qs = (InappPayment.objects.filter(id__in=ids).order_by('created').values( 'name', 'config__addon', 'created')) log.info('[%s] Indexing %s in-app payments for daily stats.' % (qs[0]['created'], len(ids))) # It's defaultdicts all the way down. addons_inapps_dates = defaultdict( lambda: defaultdict(lambda: defaultdict(int))) for payment in qs: addon = payment['config__addon'] inapp = payment['name'] date = payment['created'].strftime('%Y%m%d') # Date for add-on not processed, index it and give it key. if not date in addons_inapps_dates[addon][inapp]: key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date))) data = search.get_finance_daily_inapp(payment) try: if not already_indexed(InappPayment, data): InappPayment.index(data, bulk=True, id=key) addons_inapps_dates[addon][inapp][date] = 0 es.flush_bulk(forced=True) except Exception, exc: index_finance_daily_inapp.retry(args=[ids], exc=exc) raise
def index_finance_total_inapp_by_src(addons, **kw): """ Total finance in-app stats, src breakdown. """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() log.info('Indexing total financial in-app stats by src for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set( InappPayment.objects.filter(config__addon=addon).values_list( 'name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue # Get a list of distinct sources for given in-app. sources = set(qs.values_list('contribution__source', flat=True)) for source in sources: try: key = ord_word('srcinapp' + str(addon) + inapp_name + source.lower()) try: data = search.get_finance_total_inapp(qs, addon, inapp_name, 'source', source=source) for index in get_indices(index): if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) except Exception, e: # We ignore this error for now. See #805181 pass es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_src.retry(args=[addons], exc=exc, **kw) raise
def index_finance_total_inapp_by_currency(addons, **kw): """ Bug 758071 Total finance in-app stats, currency breakdown. """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() log.info('Indexing total financial in-app stats by currency for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set( InappPayment.objects.filter(config__addon=addon).values_list( 'name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue # Get a list of distinct currencies for given in-app. currencies = set( qs.values_list('contribution__currency', flat=True)) for currency in currencies: try: key = ord_word('curinapp' + str(addon) + inapp_name + currency.lower()) data = search.get_finance_total_inapp(qs, addon, inapp_name, 'currency', currency=currency) for index in get_indices(index): if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_currency.retry(args=[addons], exc=exc, **kw) raise
def index_finance_total_inapp_by_src(addons, **kw): """ Total finance in-app stats, src breakdown. """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() log.info('Indexing total financial in-app stats by src for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set(InappPayment.objects.filter(config__addon=addon). values_list('name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue # Get a list of distinct sources for given in-app. sources = set(qs.values_list('contribution__source', flat=True)) for source in sources: try: key = ord_word('srcinapp' + str(addon) + inapp_name + source.lower()) try: data = search.get_finance_total_inapp( qs, addon, inapp_name, 'source', source=source) for index in get_indices(index): if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) except Exception, e: # We ignore this error for now. See #805181 pass es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_src.retry(args=[addons], exc=exc, **kw) raise
def index_finance_total_inapp_by_currency(addons, **kw): """ Bug 758071 Total finance in-app stats, currency breakdown. """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() log.info('Indexing total financial in-app stats by currency for %s apps.' % len(addons)) for addon in addons: # Get all in-app names for given addon. inapps = set(InappPayment.objects.filter(config__addon=addon). values_list('name', flat=True)) for inapp_name in inapps: # Get all in-app payments for given in-app. qs = InappPayment.objects.filter(name=inapp_name, contribution__uuid=None) if not qs.exists(): continue # Get a list of distinct currencies for given in-app. currencies = set(qs.values_list('contribution__currency', flat=True)) for currency in currencies: try: key = ord_word('curinapp' + str(addon) + inapp_name + currency.lower()) data = search.get_finance_total_inapp( qs, addon, inapp_name, 'currency', currency=currency) for index in get_indices(index): if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_currency.retry(args=[addons], exc=exc, **kw) raise
def index_finance_daily_inapp(ids, **kw): """ Similar to index_finance_daily, except for InappPayments. ids -- ids of mkt.stats.webapps.InappPayment objects """ index = kw.get('index', InappPayment._get_index()) es = amo.search.get_es() # Get contributions. qs = (InappPayment.objects.filter(id__in=ids) .order_by('created').values('name', 'config__addon', 'created')) log.info('[%s] Indexing %s in-app payments for daily stats.' % (qs[0]['created'], len(ids))) # It's defaultdicts all the way down. addons_inapps_dates = defaultdict(lambda: defaultdict( lambda: defaultdict(int))) for payment in qs: addon = payment['config__addon'] inapp = payment['name'] date = payment['created'].strftime('%Y%m%d') # Date for add-on not processed, index it and give it key. if not date in addons_inapps_dates[addon][inapp]: key = ord_word('fin%s%s%s' % (str(addon), str(inapp), str(date))) data = search.get_finance_daily_inapp(payment) try: if not already_indexed(InappPayment, data, index): InappPayment.index(data, bulk=True, id=key, index=index) addons_inapps_dates[addon][inapp][date] = 0 es.flush_bulk(forced=True) except Exception, exc: index_finance_daily_inapp.retry(args=[ids], exc=exc, **kw) raise