Esempio n. 1
0
File: tasks.py Progetto: vdt/zamboni
def index_installed_daily(ids, **kw):
    """
    Takes a list of Installed ids and uses its addon and date fields to index
    stats for that day.
    ids -- ids of mkt.webapps.Installed objects
    """
    from mkt.webapps.models import Installed
    es = elasticutils.get_es()
    # Get Installed's
    qs = (Installed.objects.filter(
        id__in=set(ids)).order_by('-created').values('addon', 'created'))
    log.info('[%s] Indexing %s installed counts for daily stats.' %
             (qs[0]['created'], len(qs)))

    addons_dates = defaultdict(lambda: defaultdict(dict))
    for installed in qs:
        addon = installed['addon']
        date = installed['created'].strftime('%Y%m%d')

        try:
            if not date in addons_dates[addon]:
                key = ord_word('ins' + str(addon) + str(date))
                data = search.get_installed_daily(installed)
                if not already_indexed(Installed, data):
                    Installed.index(data, bulk=True, id=key)
                addons_dates[addon][date] = 0
            es.flush_bulk(forced=True)
        except Exception, exc:
            index_installed_daily.retry(args=[ids], exc=exc)
            raise
Esempio n. 2
0
def index_installed_daily(ids, **kw):
    """
    Takes a list of Installed ids and uses its addon and date fields to index
    stats for that day.
    ids -- ids of mkt.webapps.Installed objects
    """
    from mkt.webapps.models import Installed
    index = kw.get('index', Installed._get_index())
    es = amo.search.get_es()
    # Get Installed's
    qs = (Installed.objects.filter(id__in=set(ids)).
        order_by('-created').values('addon', 'created'))
    log.info('[%s] Indexing %s installed counts for daily stats.' %
             (qs[0]['created'], len(qs)))

    addons_dates = defaultdict(lambda: defaultdict(dict))
    for installed in qs:
        addon = installed['addon']
        date = installed['created'].strftime('%Y%m%d')

        try:
            if not date in addons_dates[addon]:
                key = ord_word('ins' + str(addon) + str(date))
                data = search.get_installed_daily(installed)
                for index in get_indices(index):

                    if not already_indexed(Installed, data, index):
                        Installed.index(data, bulk=True, id=key,
                                        index=index)
                addons_dates[addon][date] = 0
            es.flush_bulk(forced=True)
        except Exception, exc:
            index_installed_daily.retry(args=[ids], exc=exc, **kw)
            raise
Esempio n. 3
0
 def setUp(self):
     self.today = datetime.date.today()
     self.webapp = Addon.objects.get(pk=337141)
     self.user = UserProfile.objects.get(pk=999)
     self.client.login(username="******", password="******")
     self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
     Installed.index(search.extract_installed_count(self.in_), id=self.in_.pk)
     self.refresh("users_install")
Esempio n. 4
0
 def setUp(self):
     self.today = datetime.date.today()
     self.webapp = Addon.objects.get(pk=337141)
     self.user = UserProfile.objects.get(pk=999)
     self.client.login(username='******', password='******')
     self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
     installed = {'addon': self.in_.addon.id, 'created': self.in_.created}
     Installed.index(search.get_installed_daily(installed), id=self.in_.pk)
     self.refresh('users_install')
Esempio n. 5
0
 def setUp(self):
     self.today = datetime.date.today()
     self.webapp = Addon.objects.get(pk=337141)
     self.user = UserProfile.objects.get(pk=999)
     self.client.login(username='******', password='******')
     self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
     installed = {'addon': self.in_.addon.id, 'created': self.in_.created}
     Installed.index(search.get_installed_daily(installed),
                     id=self.in_.pk)
     self.refresh('users_install')
Esempio n. 6
0
def index_latest_mkt_stats(index=None, aliased=True):
    raise_if_reindex_in_progress()
    yesterday = datetime.date.today() - datetime.timedelta(days=1)

    try:
        latest = Contribution.search(index).order_by('-date').values_dict()
        latest_contribution = latest and latest[0]['date'] or yesterday
    except pyes.exceptions.SearchPhaseExecutionException:
        latest_contribution = yesterday

    try:
        latest = Installed.search(index).order_by('-date').values_dict()
        latest_install = latest and latest[0]['date'] or yesterday
    except pyes.exceptions.SearchPhaseExecutionException:
        latest_install = yesterday

    latest = min(latest_contribution, latest_install)

    fmt = lambda d: d.strftime('%Y-%m-%d')
    date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
    cron_log.info('index_mkt_stats --date=%s' % date_range)
    call_command('index_mkt_stats',
                 addons=None,
                 date=date_range,
                 index=index,
                 aliased=True)
Esempio n. 7
0
def index_installed_counts(ids, **kw):
    es = elasticutils.get_es()
    qs = Installed.objects.filter(id__in=set(ids))
    if qs:
        log.info('Indexing %s installed counts: %s'
                 % (len(qs), qs[0].created))
    try:
        for installed in qs:
            addon_id = installed.addon_id
            key = '%s-%s' % (addon_id, installed.created)
            data = search.extract_installed_count(installed)
            Installed.index(data, bulk=True, id=key)
        es.flush_bulk(forced=True)
    except Exception, exc:
        index_installed_counts.retry(args=[ids], exc=exc)
        raise
Esempio n. 8
0
def index_installed_counts(ids, **kw):
    from mkt.webapps.models import Installed

    es = elasticutils.get_es()
    qs = Installed.objects.filter(id__in=set(ids))
    if qs.exists():
        log.info('Indexing %s installed counts: %s' % (len(qs), qs[0].created))
    try:
        for installed in qs:
            addon_id = installed.addon_id
            key = '%s-%s' % (addon_id, installed.created)
            data = search.extract_installed_count(installed)
            Installed.index(data, bulk=True, id=key)
        es.flush_bulk(forced=True)
    except Exception, exc:
        index_installed_counts.retry(args=[ids], exc=exc)
        raise
Esempio n. 9
0
def index_latest_mkt_stats():
    latest_contribution = Contribution.search().order_by('-date'
        ).values_dict()[0]['date']
    latest_install = Installed.search().order_by('-date'
        ).values_dict()[0]['date']

    latest = min(latest_contribution, latest_install)

    fmt = lambda d: d.strftime('%Y-%m-%d')
    date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
    cron_log.info('index_mkt_stats --date=%s' % date_range)
    call_command('index_mkt_stats', addons=None, date=date_range)
Esempio n. 10
0
File: cron.py Progetto: vdt/zamboni
def index_latest_mkt_stats():
    latest_contribution = Contribution.search().order_by(
        '-date').values_dict()[0]['date']
    latest_install = Installed.search().order_by(
        '-date').values_dict()[0]['date']

    latest = min(latest_contribution, latest_install)

    fmt = lambda d: d.strftime('%Y-%m-%d')
    date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
    cron_log.info('index_mkt_stats --date=%s' % date_range)
    call_command('index_mkt_stats', addons=None, date=date_range)
Esempio n. 11
0
def index_latest_mkt_stats(index=None, aliased=True):
    raise_if_reindex_in_progress()
    yesterday = datetime.date.today() - datetime.timedelta(days=1)

    try:
        latest = Contribution.search(index).order_by('-date').values_dict()
        latest_contribution = latest and latest[0]['date'] or yesterday
    except pyes.exceptions.SearchPhaseExecutionException:
        latest_contribution = yesterday

    try:
        latest = Installed.search(index).order_by('-date').values_dict()
        latest_install = latest and latest[0]['date'] or yesterday
    except pyes.exceptions.SearchPhaseExecutionException:
        latest_install = yesterday

    latest = min(latest_contribution, latest_install)

    fmt = lambda d: d.strftime('%Y-%m-%d')
    date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
    cron_log.info('index_mkt_stats --date=%s' % date_range)
    call_command('index_mkt_stats', addons=None, date=date_range, index=index,
                 aliased=True)