def index_finance_total_by_currency(addons, **kw): """ Bug 757581 Total finance stats, currency breakdown. """ index = kw.get('index', Contribution._get_index()) es = amo.search.get_es() log.info('Indexing total financial stats by currency for %s apps.' % len(addons)) for addon in addons: # Get all contributions for given add-on. qs = Contribution.objects.filter(addon=addon, uuid=None) if not qs.exists(): continue # Get list of distinct currencies. currencies = set(qs.values_list('currency', flat=True)) for currency in currencies: try: key = ord_word('cur' + str(addon) + currency.lower()) data = search.get_finance_total( qs, addon, 'currency', currency=currency) for index in get_indices(index): if not already_indexed(Contribution, data, index): Contribution.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_currency.retry(args=[addons], exc=exc, **kw) raise
def index_finance_total_by_src(addons, **kw): """ Bug 758059 Total finance stats, source breakdown. """ index = kw.get('index', Contribution._get_index()) es = elasticutils.get_es() log.info('Indexing total financial stats by source for %s apps.' % len(addons)) for addon in addons: # Get all contributions for given add-on. qs = Contribution.objects.filter(addon=addon, uuid=None) if not qs.exists(): continue # Get list of distinct sources. sources = set(qs.values_list('source', flat=True)) for source in sources: try: key = ord_word('src' + str(addon) + str(source)) data = search.get_finance_total(qs, addon, 'source', source=source) for index in get_indices(index): if not already_indexed(Contribution, data, index): Contribution.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total_by_src.retry(args=[addons], exc=exc, **kw) raise
def index_finance_daily(ids, **kw): """ Bug 748015 Takes a list of Contribution ids and uses its addon and date fields to index stats for that day. Contribution stats by addon-date unique pair. Uses a nested dictionary to not index duplicate contribution with same addon/date pairs. For each addon-date, it stores the addon in the dict as a top level key with a dict as its value. And it stores the date in the add-on's dict as a second level key. To check if an addon-date pair has been already index, it looks up the dict[addon][date] to see if the key exists. This adds some speed up when batch processing. ids -- ids of apps.stats.Contribution objects """ index = kw.get('index', Contribution._get_index()) es = amo.search.get_es() # Get contributions. qs = (Contribution.objects.filter(id__in=ids).order_by('created').values( 'addon', 'created')) log.info('[%s] Indexing %s contributions for daily stats.' % (qs[0]['created'], len(ids))) addons_dates = defaultdict(lambda: defaultdict(dict)) for contribution in qs: addon = contribution['addon'] date = contribution['created'].strftime('%Y%m%d') try: # Date for add-on not processed, index it and give it key. if not date in addons_dates[addon]: key = ord_word('fin' + str(addon) + str(date)) data = search.get_finance_daily(contribution) for index in get_indices(index): if not already_indexed(Contribution, data, index): Contribution.index(data, bulk=True, id=key, index=index) addons_dates[addon][date] = 0 es.flush_bulk(forced=True) except Exception, exc: index_finance_daily.retry(args=[ids], exc=exc, **kw) raise
def index_finance_daily(ids, **kw): """ Bug 748015 Takes a list of Contribution ids and uses its addon and date fields to index stats for that day. Contribution stats by addon-date unique pair. Uses a nested dictionary to not index duplicate contribution with same addon/date pairs. For each addon-date, it stores the addon in the dict as a top level key with a dict as its value. And it stores the date in the add-on's dict as a second level key. To check if an addon-date pair has been already index, it looks up the dict[addon][date] to see if the key exists. This adds some speed up when batch processing. ids -- ids of apps.stats.Contribution objects """ index = kw.get('index', Contribution._get_index()) es = amo.search.get_es() # Get contributions. qs = (Contribution.objects.filter(id__in=ids) .order_by('created').values('addon', 'created')) log.info('[%s] Indexing %s contributions for daily stats.' % (qs[0]['created'], len(ids))) addons_dates = defaultdict(lambda: defaultdict(dict)) for contribution in qs: addon = contribution['addon'] date = contribution['created'].strftime('%Y%m%d') try: # Date for add-on not processed, index it and give it key. if not date in addons_dates[addon]: key = ord_word('fin' + str(addon) + str(date)) data = search.get_finance_daily(contribution) for index in get_indices(index): if not already_indexed(Contribution, data, index): Contribution.index(data, bulk=True, id=key, index=index) addons_dates[addon][date] = 0 es.flush_bulk(forced=True) except Exception, exc: index_finance_daily.retry(args=[ids], exc=exc, **kw) raise
def index_finance_total(addons, **kw): """ Aggregates financial stats from all of the contributions for a given app. """ index = kw.get('index', Contribution._get_index()) es = amo.search.get_es() log.info('Indexing total financial stats for %s apps.' % len(addons)) for addon in addons: # Get all contributions for given add-on. qs = Contribution.objects.filter(addon=addon, uuid=None) if not qs.exists(): continue try: key = ord_word('tot' + str(addon)) data = search.get_finance_total(qs, addon) for index in get_indices(index): if not already_indexed(Contribution, data, index): Contribution.index(data, bulk=True, id=key, index=index) es.flush_bulk(forced=True) except Exception, exc: index_finance_total.retry(args=[addons], exc=exc, **kw) raise